Launchpad itself

Merge lp://staging/~al-maisan/launchpad/empty-table-507782 into lp://staging/launchpad/db-devel

  1. empty-table-507782
  2. Merge into db-devel
Proposed by Muharem Hrnjadovic
Status: Merged
Merged at revision: not available
Proposed branch: lp://staging/~al-maisan/launchpad/empty-table-507782
Merge into: lp://staging/launchpad/db-devel
Diff against target: 359 lines (+120/-38)
8 files modified
lib/lp/buildmaster/interfaces/buildfarmjob.py (+17/-11)
lib/lp/buildmaster/model/builder.py (+3/-9)
lib/lp/buildmaster/model/buildfarmjob.py (+1/-1)
lib/lp/buildmaster/tests/test_builder.py (+66/-6)
lib/lp/soyuz/configure.zcml (+3/-0)
lib/lp/soyuz/model/buildpackagejob.py (+5/-5)
lib/lp/soyuz/model/sourcepackagerecipebuild.py (+8/-5)
lib/lp/testing/factory.py (+17/-1)
To merge this branch: bzr merge lp://staging/~al-maisan/launchpad/empty-table-507782
Reviewer Review Type Date Requested Status
Canonical Launchpad Engineering Pending
Review via email: mp+17566@code.staging.launchpad.net
To post a comment you must log in.
Revision history for this message
Muharem Hrnjadovic (al-maisan) wrote :

This branch fixes (first and foremost) the following bug: source package recipe build jobs are not seen in the absence of binary build jobs. This makes the candidate job selection fail.

Test to run:

    bin/test -vv build

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/buildmaster/interfaces/buildfarmjob.py'
2--- lib/lp/buildmaster/interfaces/buildfarmjob.py 2010-01-18 00:39:57 +0000
3+++ lib/lp/buildmaster/interfaces/buildfarmjob.py 2010-01-18 01:45:24 +0000
4@@ -165,22 +165,28 @@
5 """
6
7 def addCandidateSelectionCriteria(processor, virtualized):
8- """Provide extra clauses that will refine the candidate job selection.
9-
10- Return a 2-tuple with extra tables and clauses to be used to
11- narrow down the list of candidate jobs.
12-
13- Example:
14- (('Build', 'BuildPackageJob'),
15- "BuildPackageJob.build = Build.id AND ..")
16+ """Provide a sub-query to refine the candidate job selection.
17+
18+ Return a sub-query to narrow down the list of candidate jobs.
19+ The sub-query will become part of an "outer query" and is free to
20+ refer to the `BuildQueue` and `Job` tables already utilized in the
21+ latter.
22+
23+ Example (please see the `BuildPackageJob` implementation for a
24+ complete example):
25+
26+ SELECT TRUE
27+ FROM Archive, Build, BuildPackageJob, DistroArchSeries
28+ WHERE
29+ BuildPackageJob.job = Job.id AND
30+ ..
31
32 :param processor: the type of processor that the candidate jobs are
33 expected to run on.
34 :param virtualized: whether the candidate jobs are expected to run on
35 the `processor` natively or inside a virtual machine.
36- :return: an (extra_tables, extra_query) tuple where `extra_tables` is
37- a collection of tables that need to appear in the FROM clause of
38- the combined query for `extra_query` to work.
39+ :return: a string containing a sub-query that narrows down the list of
40+ candidate jobs.
41 """
42
43 def postprocessCandidate(job, logger):
44
45=== modified file 'lib/lp/buildmaster/model/builder.py'
46--- lib/lp/buildmaster/model/builder.py 2010-01-16 05:54:16 +0000
47+++ lib/lp/buildmaster/model/builder.py 2010-01-18 01:45:24 +0000
48@@ -437,7 +437,7 @@
49 def qualify_subquery(job_type, sub_query):
50 """Put the sub-query into a job type context."""
51 qualified_query = """
52- ((BuildQueue.job_type != %s) OR (%%s))
53+ ((BuildQueue.job_type != %s) OR EXISTS(%%s))
54 """ % sqlvalues(job_type)
55 qualified_query %= sub_query
56 return qualified_query
57@@ -445,9 +445,8 @@
58 logger = self._getSlaveScannerLogger()
59 candidate = None
60
61- query_tables = set(('buildqueue', 'job'))
62 general_query = """
63- SELECT buildqueue.id FROM %%s
64+ SELECT buildqueue.id FROM buildqueue, job
65 WHERE
66 buildqueue.job = job.id
67 AND job.status = %s
68@@ -474,20 +473,15 @@
69 extra_queries = []
70 job_classes = specific_job_classes()
71 for job_type, job_class in job_classes.iteritems():
72- tables, query = job_class.addCandidateSelectionCriteria(
73+ query = job_class.addCandidateSelectionCriteria(
74 self.processor, self.virtualized)
75 if query == '':
76 # This job class does not need to refine candidate jobs
77 # further.
78 continue
79
80- # Table names are case-insensitive in SQL. All table names are in
81- # lower case in order to avoid duplicates in the FROM clause.
82- query_tables = query_tables.union(
83- set(table.lower() for table in tables))
84 # The sub-query should only apply to jobs of the right type.
85 extra_queries.append(qualify_subquery(job_type, query))
86- general_query = general_query % ', '.join(query_tables)
87 query = ' AND '.join([general_query] + extra_queries) + order_clause
88
89 store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
90
91=== modified file 'lib/lp/buildmaster/model/buildfarmjob.py'
92--- lib/lp/buildmaster/model/buildfarmjob.py 2010-01-16 07:52:28 +0000
93+++ lib/lp/buildmaster/model/buildfarmjob.py 2010-01-18 01:45:24 +0000
94@@ -62,7 +62,7 @@
95 @staticmethod
96 def addCandidateSelectionCriteria(processor, virtualized):
97 """See `IBuildFarmCandidateJobSelection`."""
98- return ([], '')
99+ return ('')
100
101 @classmethod
102 def getByJob(cls, job):
103
104=== modified file 'lib/lp/buildmaster/tests/test_builder.py'
105--- lib/lp/buildmaster/tests/test_builder.py 2010-01-12 20:26:34 +0000
106+++ lib/lp/buildmaster/tests/test_builder.py 2010-01-18 01:45:24 +0000
107@@ -8,16 +8,22 @@
108 from zope.component import getUtility
109 from zope.security.proxy import removeSecurityProxy
110
111+from canonical.launchpad.webapp.interfaces import (
112+ IStoreSelector, MAIN_STORE, DEFAULT_FLAVOR)
113 from canonical.testing import LaunchpadZopelessLayer
114+from lp.buildmaster.interfaces.builder import IBuilderSet
115+from lp.buildmaster.interfaces.buildfarmjob import BuildFarmJobType
116 from lp.buildmaster.interfaces.buildfarmjobbehavior import (
117 IBuildFarmJobBehavior)
118 from lp.buildmaster.model.buildfarmjobbehavior import IdleBuildBehavior
119 from lp.soyuz.interfaces.archive import ArchivePurpose
120 from lp.soyuz.interfaces.build import BuildStatus, IBuildSet
121-from lp.buildmaster.interfaces.builder import IBuilderSet
122 from lp.soyuz.interfaces.publishing import PackagePublishingStatus
123+from lp.soyuz.interfaces.sourcepackagerecipebuild import (
124+ ISourcePackageRecipeBuildSource)
125 from lp.soyuz.model.binarypackagebuildbehavior import (
126 BinaryPackageBuildBehavior)
127+from lp.soyuz.model.buildqueue import BuildQueue
128 from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
129 from lp.testing import TestCaseWithFactory
130
131@@ -194,15 +200,15 @@
132 super(TestFindBuildCandidateDistroArchive, self).setUp()
133 # Create a primary archive and publish some builds for the
134 # queue.
135- non_ppa = self.factory.makeArchive(
136+ self.non_ppa = self.factory.makeArchive(
137 name="primary", purpose=ArchivePurpose.PRIMARY)
138
139- gedit_build = self.publisher.getPubSource(
140+ self.gedit_build = self.publisher.getPubSource(
141 sourcename="gedit", status=PackagePublishingStatus.PUBLISHED,
142- archive=non_ppa).createMissingBuilds()[0]
143- firefox_build = self.publisher.getPubSource(
144+ archive=self.non_ppa).createMissingBuilds()[0]
145+ self.firefox_build = self.publisher.getPubSource(
146 sourcename="firefox", status=PackagePublishingStatus.PUBLISHED,
147- archive=non_ppa).createMissingBuilds()[0]
148+ archive=self.non_ppa).createMissingBuilds()[0]
149
150 def test_findBuildCandidate_for_non_ppa(self):
151 # Normal archives are not restricted to serial builds per
152@@ -224,6 +230,60 @@
153 self.failUnlessEqual('primary', build.archive.name)
154 self.failUnlessEqual('firefox', build.sourcepackagerelease.name)
155
156+ def test_findBuildCandidate_for_recipe_build(self):
157+ # Recipe builds with a higher score are selected first.
158+ # This test is run in a context with mixed recipe and binary builds.
159+
160+ self.assertIsNot(self.frog_builder.processor, None)
161+ self.assertEqual(self.frog_builder.virtualized, True)
162+
163+ self.assertEqual(self.gedit_build.buildqueue_record.lastscore, 2505)
164+ self.assertEqual(self.firefox_build.buildqueue_record.lastscore, 2505)
165+
166+ recipe_build_job = self.factory.makeSourcePackageRecipeBuildJob(9999)
167+
168+ self.assertEqual(recipe_build_job.lastscore, 9999)
169+
170+ next_job = removeSecurityProxy(
171+ self.frog_builder)._findBuildCandidate()
172+
173+ self.failUnlessEqual(recipe_build_job, next_job)
174+
175+
176+class TestFindRecipeBuildCandidates(TestFindBuildCandidateBase):
177+ # These tests operate in a "recipe builds only" setting.
178+ # Please see also bug #507782.
179+
180+ def clearBuildQueue(self):
181+ """Delete all `BuildQueue`, XXXJOb and `Job` instances."""
182+ store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
183+ for bq in store.find(BuildQueue):
184+ bq.destroySelf()
185+
186+ def setUp(self):
187+ """Publish some builds for the test archive."""
188+ super(TestFindRecipeBuildCandidates, self).setUp()
189+ # Create a primary archive and publish some builds for the
190+ # queue.
191+ self.non_ppa = self.factory.makeArchive(
192+ name="primary", purpose=ArchivePurpose.PRIMARY)
193+
194+ self.clearBuildQueue()
195+ self.bq1 = self.factory.makeSourcePackageRecipeBuildJob(3333)
196+ self.bq2 = self.factory.makeSourcePackageRecipeBuildJob(4333)
197+
198+ def test_findBuildCandidate_with_highest_score(self):
199+ # The recipe build with the highest score is selected first.
200+ # This test is run in a "recipe builds only" context.
201+
202+ self.assertIsNot(self.frog_builder.processor, None)
203+ self.assertEqual(self.frog_builder.virtualized, True)
204+
205+ next_job = removeSecurityProxy(
206+ self.frog_builder)._findBuildCandidate()
207+
208+ self.failUnlessEqual(self.bq2, next_job)
209+
210
211 class TestCurrentBuildBehavior(TestCaseWithFactory):
212 """This test ensures the get/set behavior of IBuilder's
213
214=== modified file 'lib/lp/soyuz/configure.zcml'
215--- lib/lp/soyuz/configure.zcml 2010-01-17 05:01:30 +0000
216+++ lib/lp/soyuz/configure.zcml 2010-01-18 01:45:24 +0000
217@@ -962,6 +962,9 @@
218 class="lp.soyuz.model.sourcepackagerecipebuild.SourcePackageRecipeBuildJob">
219 <allow interface="lp.soyuz.interfaces.sourcepackagerecipebuild.ISourcePackageRecipeBuildJob"/>
220 </class>
221+ <utility component="lp.soyuz.model.sourcepackagerecipebuild.SourcePackageRecipeBuildJob"
222+ name="RECIPEBRANCHBUILD"
223+ provides="lp.buildmaster.interfaces.buildfarmjob.IBuildFarmJob"/>
224
225 <securedutility
226 component="lp.soyuz.model.sourcepackagerecipebuild.SourcePackageRecipeBuildJob"
227
228=== modified file 'lib/lp/soyuz/model/buildpackagejob.py'
229--- lib/lp/soyuz/model/buildpackagejob.py 2010-01-16 09:45:58 +0000
230+++ lib/lp/soyuz/model/buildpackagejob.py 2010-01-18 01:45:24 +0000
231@@ -230,9 +230,9 @@
232 PackagePublishingStatus.SUPERSEDED,
233 PackagePublishingStatus.DELETED,
234 )
235- extra_tables = [
236- 'Archive', 'Build', 'BuildPackageJob', 'DistroArchSeries']
237- extra_clauses = """
238+ sub_query = """
239+ SELECT TRUE FROM Archive, Build, BuildPackageJob, DistroArchSeries
240+ WHERE
241 BuildPackageJob.job = Job.id AND
242 BuildPackageJob.build = Build.id AND
243 Build.distroarchseries = DistroArchSeries.id AND
244@@ -269,7 +269,7 @@
245 num_arch_builders = Builder.selectBy(
246 processor=processor, manual=False, builderok=True).count()
247 if num_arch_builders > 1:
248- extra_clauses += """
249+ sub_query += """
250 AND EXISTS (SELECT true
251 WHERE ((
252 SELECT COUNT(build2.id)
253@@ -287,7 +287,7 @@
254 ArchivePurpose.PPA, processor.family,
255 BuildStatus.BUILDING, num_arch_builders)
256
257- return(extra_tables, extra_clauses)
258+ return sub_query
259
260 @staticmethod
261 def postprocessCandidate(job, logger):
262
263=== modified file 'lib/lp/soyuz/model/sourcepackagerecipebuild.py'
264--- lib/lp/soyuz/model/sourcepackagerecipebuild.py 2010-01-18 00:39:57 +0000
265+++ lib/lp/soyuz/model/sourcepackagerecipebuild.py 2010-01-18 01:45:24 +0000
266@@ -19,8 +19,9 @@
267 from zope.component import getUtility
268 from zope.interface import classProvides, implements
269
270+from lp.buildmaster.model.buildfarmjob import BuildFarmJob
271+from lp.registry.interfaces.pocket import PackagePublishingPocket
272 from lp.services.job.model.job import Job
273-from lp.registry.interfaces.pocket import PackagePublishingPocket
274 from lp.soyuz.adapters.archivedependencies import (
275 default_component_dependency_name,)
276 from lp.soyuz.interfaces.build import BuildStatus
277@@ -65,7 +66,8 @@
278 distroseries = Reference(distroseries_id, 'DistroSeries.id')
279
280 sourcepackagename_id = Int(name='sourcepackagename', allow_none=True)
281- sourcepackagename = Reference(sourcepackagename_id, 'SourcePackageName.id')
282+ sourcepackagename = Reference(
283+ sourcepackagename_id, 'SourcePackageName.id')
284
285 @property
286 def pocket(self):
287@@ -101,7 +103,8 @@
288 self.sourcepackagename = sourcepackagename
289
290 @classmethod
291- def new(cls, sourcepackage, recipe, requester, archive, date_created=None):
292+ def new(
293+ cls, sourcepackage, recipe, requester, archive, date_created=None):
294 """See `ISourcePackageRecipeBuildSource`."""
295 store = IMasterStore(SourcePackageRecipeBuild)
296 if date_created is None:
297@@ -126,7 +129,7 @@
298 return specific_job
299
300
301-class SourcePackageRecipeBuildJob(Storm):
302+class SourcePackageRecipeBuildJob(BuildFarmJob, Storm):
303
304 classProvides(ISourcePackageRecipeBuildJobSource)
305 implements(ISourcePackageRecipeBuildJob)
306@@ -143,7 +146,7 @@
307 source_package_build_id, 'SourcePackageRecipeBuild.id')
308
309 processor = None
310- virtualized = False
311+ virtualized = True
312
313 def __init__(self, build, job):
314 super(SourcePackageRecipeBuildJob, self).__init__()
315
316=== modified file 'lib/lp/testing/factory.py'
317--- lib/lp/testing/factory.py 2010-01-17 23:32:21 +0000
318+++ lib/lp/testing/factory.py 2010-01-18 01:45:24 +0000
319@@ -86,7 +86,8 @@
320 from lp.services.mail.signedmessage import SignedMessage
321 from lp.services.worlddata.interfaces.country import ICountrySet
322 from canonical.launchpad.webapp.dbpolicy import MasterDatabasePolicy
323-from canonical.launchpad.webapp.interfaces import IStoreSelector
324+from canonical.launchpad.webapp.interfaces import (
325+ IStoreSelector, MAIN_STORE, DEFAULT_FLAVOR)
326 from lp.code.enums import (
327 BranchMergeProposalStatus, BranchSubscriptionNotificationLevel,
328 BranchType, CodeImportMachineState, CodeImportReviewStatus,
329@@ -125,8 +126,10 @@
330 from lp.registry.interfaces.ssh import ISSHKeySet, SSHKeyType
331 from lp.services.worlddata.interfaces.language import ILanguageSet
332 from lp.buildmaster.interfaces.builder import IBuilderSet
333+from lp.buildmaster.interfaces.buildfarmjob import BuildFarmJobType
334 from lp.soyuz.interfaces.component import IComponentSet
335 from lp.soyuz.interfaces.packageset import IPackagesetSet
336+from lp.soyuz.model.buildqueue import BuildQueue
337 from lp.testing import run_with_login, time_counter
338
339 SPACE = ' '
340@@ -1615,6 +1618,19 @@
341 archive=archive,
342 requester=requester)
343
344+ def makeSourcePackageRecipeBuildJob(self, score=9876):
345+ """Create a `SourcePackageRecipeBuildJob` and a `BuildQueue` for
346+ testing."""
347+ recipe_build = self.makeSourcePackageRecipeBuild()
348+ recipe_build_job = recipe_build.makeJob()
349+
350+ store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
351+ bq = BuildQueue(
352+ job=recipe_build_job.job, lastscore=score,
353+ job_type=BuildFarmJobType.RECIPEBRANCHBUILD)
354+ store.add(bq)
355+ return bq
356+
357 def makePOTemplate(self, productseries=None, distroseries=None,
358 sourcepackagename=None, owner=None, name=None,
359 translation_domain=None, path=None):

Subscribers

People subscribed via source and target branches

to status/vote changes: