Launchpad itself

Merge lp://staging/~al-maisan/launchpad/empty-table-507782 into lp://staging/launchpad/db-devel

  1. empty-table-507782
  2. Merge into db-devel
Proposed by Muharem Hrnjadovic
Status: Merged
Merged at revision: not available
Proposed branch: lp://staging/~al-maisan/launchpad/empty-table-507782
Merge into: lp://staging/launchpad/db-devel
Diff against target: 359 lines (+120/-38)
8 files modified
lib/lp/buildmaster/interfaces/buildfarmjob.py (+17/-11)
lib/lp/buildmaster/model/builder.py (+3/-9)
lib/lp/buildmaster/model/buildfarmjob.py (+1/-1)
lib/lp/buildmaster/tests/test_builder.py (+66/-6)
lib/lp/soyuz/configure.zcml (+3/-0)
lib/lp/soyuz/model/buildpackagejob.py (+5/-5)
lib/lp/soyuz/model/sourcepackagerecipebuild.py (+8/-5)
lib/lp/testing/factory.py (+17/-1)
To merge this branch: bzr merge lp://staging/~al-maisan/launchpad/empty-table-507782
Reviewer Review Type Date Requested Status
Canonical Launchpad Engineering Pending
Review via email: mp+17566@code.staging.launchpad.net
To post a comment you must log in.
Revision history for this message
Muharem Hrnjadovic (al-maisan) wrote :

This branch fixes (first and foremost) the following bug: source package recipe build jobs are not seen in the absence of binary build jobs. This makes the candidate job selection fail.

Test to run:

    bin/test -vv build

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'lib/lp/buildmaster/interfaces/buildfarmjob.py'
--- lib/lp/buildmaster/interfaces/buildfarmjob.py 2010-01-18 00:39:57 +0000
+++ lib/lp/buildmaster/interfaces/buildfarmjob.py 2010-01-18 01:45:24 +0000
@@ -165,22 +165,28 @@
165 """165 """
166166
167 def addCandidateSelectionCriteria(processor, virtualized):167 def addCandidateSelectionCriteria(processor, virtualized):
168 """Provide extra clauses that will refine the candidate job selection.168 """Provide a sub-query to refine the candidate job selection.
169169
170 Return a 2-tuple with extra tables and clauses to be used to170 Return a sub-query to narrow down the list of candidate jobs.
171 narrow down the list of candidate jobs.171 The sub-query will become part of an "outer query" and is free to
172172 refer to the `BuildQueue` and `Job` tables already utilized in the
173 Example:173 latter.
174 (('Build', 'BuildPackageJob'),174
175 "BuildPackageJob.build = Build.id AND ..")175 Example (please see the `BuildPackageJob` implementation for a
176 complete example):
177
178 SELECT TRUE
179 FROM Archive, Build, BuildPackageJob, DistroArchSeries
180 WHERE
181 BuildPackageJob.job = Job.id AND
182 ..
176183
177 :param processor: the type of processor that the candidate jobs are184 :param processor: the type of processor that the candidate jobs are
178 expected to run on.185 expected to run on.
179 :param virtualized: whether the candidate jobs are expected to run on186 :param virtualized: whether the candidate jobs are expected to run on
180 the `processor` natively or inside a virtual machine.187 the `processor` natively or inside a virtual machine.
181 :return: an (extra_tables, extra_query) tuple where `extra_tables` is188 :return: a string containing a sub-query that narrows down the list of
182 a collection of tables that need to appear in the FROM clause of189 candidate jobs.
183 the combined query for `extra_query` to work.
184 """190 """
185191
186 def postprocessCandidate(job, logger):192 def postprocessCandidate(job, logger):
187193
=== modified file 'lib/lp/buildmaster/model/builder.py'
--- lib/lp/buildmaster/model/builder.py 2010-01-16 05:54:16 +0000
+++ lib/lp/buildmaster/model/builder.py 2010-01-18 01:45:24 +0000
@@ -437,7 +437,7 @@
437 def qualify_subquery(job_type, sub_query):437 def qualify_subquery(job_type, sub_query):
438 """Put the sub-query into a job type context."""438 """Put the sub-query into a job type context."""
439 qualified_query = """439 qualified_query = """
440 ((BuildQueue.job_type != %s) OR (%%s))440 ((BuildQueue.job_type != %s) OR EXISTS(%%s))
441 """ % sqlvalues(job_type)441 """ % sqlvalues(job_type)
442 qualified_query %= sub_query442 qualified_query %= sub_query
443 return qualified_query443 return qualified_query
@@ -445,9 +445,8 @@
445 logger = self._getSlaveScannerLogger()445 logger = self._getSlaveScannerLogger()
446 candidate = None446 candidate = None
447447
448 query_tables = set(('buildqueue', 'job'))
449 general_query = """448 general_query = """
450 SELECT buildqueue.id FROM %%s449 SELECT buildqueue.id FROM buildqueue, job
451 WHERE450 WHERE
452 buildqueue.job = job.id451 buildqueue.job = job.id
453 AND job.status = %s452 AND job.status = %s
@@ -474,20 +473,15 @@
474 extra_queries = []473 extra_queries = []
475 job_classes = specific_job_classes()474 job_classes = specific_job_classes()
476 for job_type, job_class in job_classes.iteritems():475 for job_type, job_class in job_classes.iteritems():
477 tables, query = job_class.addCandidateSelectionCriteria(476 query = job_class.addCandidateSelectionCriteria(
478 self.processor, self.virtualized)477 self.processor, self.virtualized)
479 if query == '':478 if query == '':
480 # This job class does not need to refine candidate jobs479 # This job class does not need to refine candidate jobs
481 # further.480 # further.
482 continue481 continue
483482
484 # Table names are case-insensitive in SQL. All table names are in
485 # lower case in order to avoid duplicates in the FROM clause.
486 query_tables = query_tables.union(
487 set(table.lower() for table in tables))
488 # The sub-query should only apply to jobs of the right type.483 # The sub-query should only apply to jobs of the right type.
489 extra_queries.append(qualify_subquery(job_type, query))484 extra_queries.append(qualify_subquery(job_type, query))
490 general_query = general_query % ', '.join(query_tables)
491 query = ' AND '.join([general_query] + extra_queries) + order_clause485 query = ' AND '.join([general_query] + extra_queries) + order_clause
492486
493 store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)487 store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
494488
=== modified file 'lib/lp/buildmaster/model/buildfarmjob.py'
--- lib/lp/buildmaster/model/buildfarmjob.py 2010-01-16 07:52:28 +0000
+++ lib/lp/buildmaster/model/buildfarmjob.py 2010-01-18 01:45:24 +0000
@@ -62,7 +62,7 @@
62 @staticmethod62 @staticmethod
63 def addCandidateSelectionCriteria(processor, virtualized):63 def addCandidateSelectionCriteria(processor, virtualized):
64 """See `IBuildFarmCandidateJobSelection`."""64 """See `IBuildFarmCandidateJobSelection`."""
65 return ([], '')65 return ('')
6666
67 @classmethod67 @classmethod
68 def getByJob(cls, job):68 def getByJob(cls, job):
6969
=== modified file 'lib/lp/buildmaster/tests/test_builder.py'
--- lib/lp/buildmaster/tests/test_builder.py 2010-01-12 20:26:34 +0000
+++ lib/lp/buildmaster/tests/test_builder.py 2010-01-18 01:45:24 +0000
@@ -8,16 +8,22 @@
8from zope.component import getUtility8from zope.component import getUtility
9from zope.security.proxy import removeSecurityProxy9from zope.security.proxy import removeSecurityProxy
1010
11from canonical.launchpad.webapp.interfaces import (
12 IStoreSelector, MAIN_STORE, DEFAULT_FLAVOR)
11from canonical.testing import LaunchpadZopelessLayer13from canonical.testing import LaunchpadZopelessLayer
14from lp.buildmaster.interfaces.builder import IBuilderSet
15from lp.buildmaster.interfaces.buildfarmjob import BuildFarmJobType
12from lp.buildmaster.interfaces.buildfarmjobbehavior import (16from lp.buildmaster.interfaces.buildfarmjobbehavior import (
13 IBuildFarmJobBehavior)17 IBuildFarmJobBehavior)
14from lp.buildmaster.model.buildfarmjobbehavior import IdleBuildBehavior18from lp.buildmaster.model.buildfarmjobbehavior import IdleBuildBehavior
15from lp.soyuz.interfaces.archive import ArchivePurpose19from lp.soyuz.interfaces.archive import ArchivePurpose
16from lp.soyuz.interfaces.build import BuildStatus, IBuildSet20from lp.soyuz.interfaces.build import BuildStatus, IBuildSet
17from lp.buildmaster.interfaces.builder import IBuilderSet
18from lp.soyuz.interfaces.publishing import PackagePublishingStatus21from lp.soyuz.interfaces.publishing import PackagePublishingStatus
22from lp.soyuz.interfaces.sourcepackagerecipebuild import (
23 ISourcePackageRecipeBuildSource)
19from lp.soyuz.model.binarypackagebuildbehavior import (24from lp.soyuz.model.binarypackagebuildbehavior import (
20 BinaryPackageBuildBehavior)25 BinaryPackageBuildBehavior)
26from lp.soyuz.model.buildqueue import BuildQueue
21from lp.soyuz.tests.test_publishing import SoyuzTestPublisher27from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
22from lp.testing import TestCaseWithFactory28from lp.testing import TestCaseWithFactory
2329
@@ -194,15 +200,15 @@
194 super(TestFindBuildCandidateDistroArchive, self).setUp()200 super(TestFindBuildCandidateDistroArchive, self).setUp()
195 # Create a primary archive and publish some builds for the201 # Create a primary archive and publish some builds for the
196 # queue.202 # queue.
197 non_ppa = self.factory.makeArchive(203 self.non_ppa = self.factory.makeArchive(
198 name="primary", purpose=ArchivePurpose.PRIMARY)204 name="primary", purpose=ArchivePurpose.PRIMARY)
199205
200 gedit_build = self.publisher.getPubSource(206 self.gedit_build = self.publisher.getPubSource(
201 sourcename="gedit", status=PackagePublishingStatus.PUBLISHED,207 sourcename="gedit", status=PackagePublishingStatus.PUBLISHED,
202 archive=non_ppa).createMissingBuilds()[0]208 archive=self.non_ppa).createMissingBuilds()[0]
203 firefox_build = self.publisher.getPubSource(209 self.firefox_build = self.publisher.getPubSource(
204 sourcename="firefox", status=PackagePublishingStatus.PUBLISHED,210 sourcename="firefox", status=PackagePublishingStatus.PUBLISHED,
205 archive=non_ppa).createMissingBuilds()[0]211 archive=self.non_ppa).createMissingBuilds()[0]
206212
207 def test_findBuildCandidate_for_non_ppa(self):213 def test_findBuildCandidate_for_non_ppa(self):
208 # Normal archives are not restricted to serial builds per214 # Normal archives are not restricted to serial builds per
@@ -224,6 +230,60 @@
224 self.failUnlessEqual('primary', build.archive.name)230 self.failUnlessEqual('primary', build.archive.name)
225 self.failUnlessEqual('firefox', build.sourcepackagerelease.name)231 self.failUnlessEqual('firefox', build.sourcepackagerelease.name)
226232
233 def test_findBuildCandidate_for_recipe_build(self):
234 # Recipe builds with a higher score are selected first.
235 # This test is run in a context with mixed recipe and binary builds.
236
237 self.assertIsNot(self.frog_builder.processor, None)
238 self.assertEqual(self.frog_builder.virtualized, True)
239
240 self.assertEqual(self.gedit_build.buildqueue_record.lastscore, 2505)
241 self.assertEqual(self.firefox_build.buildqueue_record.lastscore, 2505)
242
243 recipe_build_job = self.factory.makeSourcePackageRecipeBuildJob(9999)
244
245 self.assertEqual(recipe_build_job.lastscore, 9999)
246
247 next_job = removeSecurityProxy(
248 self.frog_builder)._findBuildCandidate()
249
250 self.failUnlessEqual(recipe_build_job, next_job)
251
252
253class TestFindRecipeBuildCandidates(TestFindBuildCandidateBase):
254 # These tests operate in a "recipe builds only" setting.
255 # Please see also bug #507782.
256
257 def clearBuildQueue(self):
258 """Delete all `BuildQueue`, XXXJOb and `Job` instances."""
259 store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
260 for bq in store.find(BuildQueue):
261 bq.destroySelf()
262
263 def setUp(self):
264 """Publish some builds for the test archive."""
265 super(TestFindRecipeBuildCandidates, self).setUp()
266 # Create a primary archive and publish some builds for the
267 # queue.
268 self.non_ppa = self.factory.makeArchive(
269 name="primary", purpose=ArchivePurpose.PRIMARY)
270
271 self.clearBuildQueue()
272 self.bq1 = self.factory.makeSourcePackageRecipeBuildJob(3333)
273 self.bq2 = self.factory.makeSourcePackageRecipeBuildJob(4333)
274
275 def test_findBuildCandidate_with_highest_score(self):
276 # The recipe build with the highest score is selected first.
277 # This test is run in a "recipe builds only" context.
278
279 self.assertIsNot(self.frog_builder.processor, None)
280 self.assertEqual(self.frog_builder.virtualized, True)
281
282 next_job = removeSecurityProxy(
283 self.frog_builder)._findBuildCandidate()
284
285 self.failUnlessEqual(self.bq2, next_job)
286
227287
228class TestCurrentBuildBehavior(TestCaseWithFactory):288class TestCurrentBuildBehavior(TestCaseWithFactory):
229 """This test ensures the get/set behavior of IBuilder's289 """This test ensures the get/set behavior of IBuilder's
230290
=== modified file 'lib/lp/soyuz/configure.zcml'
--- lib/lp/soyuz/configure.zcml 2010-01-17 05:01:30 +0000
+++ lib/lp/soyuz/configure.zcml 2010-01-18 01:45:24 +0000
@@ -962,6 +962,9 @@
962 class="lp.soyuz.model.sourcepackagerecipebuild.SourcePackageRecipeBuildJob">962 class="lp.soyuz.model.sourcepackagerecipebuild.SourcePackageRecipeBuildJob">
963 <allow interface="lp.soyuz.interfaces.sourcepackagerecipebuild.ISourcePackageRecipeBuildJob"/>963 <allow interface="lp.soyuz.interfaces.sourcepackagerecipebuild.ISourcePackageRecipeBuildJob"/>
964 </class>964 </class>
965 <utility component="lp.soyuz.model.sourcepackagerecipebuild.SourcePackageRecipeBuildJob"
966 name="RECIPEBRANCHBUILD"
967 provides="lp.buildmaster.interfaces.buildfarmjob.IBuildFarmJob"/>
965968
966 <securedutility969 <securedutility
967 component="lp.soyuz.model.sourcepackagerecipebuild.SourcePackageRecipeBuildJob"970 component="lp.soyuz.model.sourcepackagerecipebuild.SourcePackageRecipeBuildJob"
968971
=== modified file 'lib/lp/soyuz/model/buildpackagejob.py'
--- lib/lp/soyuz/model/buildpackagejob.py 2010-01-16 09:45:58 +0000
+++ lib/lp/soyuz/model/buildpackagejob.py 2010-01-18 01:45:24 +0000
@@ -230,9 +230,9 @@
230 PackagePublishingStatus.SUPERSEDED,230 PackagePublishingStatus.SUPERSEDED,
231 PackagePublishingStatus.DELETED,231 PackagePublishingStatus.DELETED,
232 )232 )
233 extra_tables = [233 sub_query = """
234 'Archive', 'Build', 'BuildPackageJob', 'DistroArchSeries']234 SELECT TRUE FROM Archive, Build, BuildPackageJob, DistroArchSeries
235 extra_clauses = """235 WHERE
236 BuildPackageJob.job = Job.id AND 236 BuildPackageJob.job = Job.id AND
237 BuildPackageJob.build = Build.id AND 237 BuildPackageJob.build = Build.id AND
238 Build.distroarchseries = DistroArchSeries.id AND238 Build.distroarchseries = DistroArchSeries.id AND
@@ -269,7 +269,7 @@
269 num_arch_builders = Builder.selectBy(269 num_arch_builders = Builder.selectBy(
270 processor=processor, manual=False, builderok=True).count()270 processor=processor, manual=False, builderok=True).count()
271 if num_arch_builders > 1:271 if num_arch_builders > 1:
272 extra_clauses += """272 sub_query += """
273 AND EXISTS (SELECT true273 AND EXISTS (SELECT true
274 WHERE ((274 WHERE ((
275 SELECT COUNT(build2.id)275 SELECT COUNT(build2.id)
@@ -287,7 +287,7 @@
287 ArchivePurpose.PPA, processor.family,287 ArchivePurpose.PPA, processor.family,
288 BuildStatus.BUILDING, num_arch_builders)288 BuildStatus.BUILDING, num_arch_builders)
289289
290 return(extra_tables, extra_clauses)290 return sub_query
291291
292 @staticmethod292 @staticmethod
293 def postprocessCandidate(job, logger):293 def postprocessCandidate(job, logger):
294294
=== modified file 'lib/lp/soyuz/model/sourcepackagerecipebuild.py'
--- lib/lp/soyuz/model/sourcepackagerecipebuild.py 2010-01-18 00:39:57 +0000
+++ lib/lp/soyuz/model/sourcepackagerecipebuild.py 2010-01-18 01:45:24 +0000
@@ -19,8 +19,9 @@
19from zope.component import getUtility19from zope.component import getUtility
20from zope.interface import classProvides, implements20from zope.interface import classProvides, implements
2121
22from lp.buildmaster.model.buildfarmjob import BuildFarmJob
23from lp.registry.interfaces.pocket import PackagePublishingPocket
22from lp.services.job.model.job import Job24from lp.services.job.model.job import Job
23from lp.registry.interfaces.pocket import PackagePublishingPocket
24from lp.soyuz.adapters.archivedependencies import (25from lp.soyuz.adapters.archivedependencies import (
25 default_component_dependency_name,)26 default_component_dependency_name,)
26from lp.soyuz.interfaces.build import BuildStatus27from lp.soyuz.interfaces.build import BuildStatus
@@ -65,7 +66,8 @@
65 distroseries = Reference(distroseries_id, 'DistroSeries.id')66 distroseries = Reference(distroseries_id, 'DistroSeries.id')
6667
67 sourcepackagename_id = Int(name='sourcepackagename', allow_none=True)68 sourcepackagename_id = Int(name='sourcepackagename', allow_none=True)
68 sourcepackagename = Reference(sourcepackagename_id, 'SourcePackageName.id')69 sourcepackagename = Reference(
70 sourcepackagename_id, 'SourcePackageName.id')
6971
70 @property72 @property
71 def pocket(self):73 def pocket(self):
@@ -101,7 +103,8 @@
101 self.sourcepackagename = sourcepackagename103 self.sourcepackagename = sourcepackagename
102104
103 @classmethod105 @classmethod
104 def new(cls, sourcepackage, recipe, requester, archive, date_created=None):106 def new(
107 cls, sourcepackage, recipe, requester, archive, date_created=None):
105 """See `ISourcePackageRecipeBuildSource`."""108 """See `ISourcePackageRecipeBuildSource`."""
106 store = IMasterStore(SourcePackageRecipeBuild)109 store = IMasterStore(SourcePackageRecipeBuild)
107 if date_created is None:110 if date_created is None:
@@ -126,7 +129,7 @@
126 return specific_job129 return specific_job
127130
128131
129class SourcePackageRecipeBuildJob(Storm):132class SourcePackageRecipeBuildJob(BuildFarmJob, Storm):
130133
131 classProvides(ISourcePackageRecipeBuildJobSource)134 classProvides(ISourcePackageRecipeBuildJobSource)
132 implements(ISourcePackageRecipeBuildJob)135 implements(ISourcePackageRecipeBuildJob)
@@ -143,7 +146,7 @@
143 source_package_build_id, 'SourcePackageRecipeBuild.id')146 source_package_build_id, 'SourcePackageRecipeBuild.id')
144147
145 processor = None148 processor = None
146 virtualized = False149 virtualized = True
147150
148 def __init__(self, build, job):151 def __init__(self, build, job):
149 super(SourcePackageRecipeBuildJob, self).__init__()152 super(SourcePackageRecipeBuildJob, self).__init__()
150153
=== modified file 'lib/lp/testing/factory.py'
--- lib/lp/testing/factory.py 2010-01-17 23:32:21 +0000
+++ lib/lp/testing/factory.py 2010-01-18 01:45:24 +0000
@@ -86,7 +86,8 @@
86from lp.services.mail.signedmessage import SignedMessage86from lp.services.mail.signedmessage import SignedMessage
87from lp.services.worlddata.interfaces.country import ICountrySet87from lp.services.worlddata.interfaces.country import ICountrySet
88from canonical.launchpad.webapp.dbpolicy import MasterDatabasePolicy88from canonical.launchpad.webapp.dbpolicy import MasterDatabasePolicy
89from canonical.launchpad.webapp.interfaces import IStoreSelector89from canonical.launchpad.webapp.interfaces import (
90 IStoreSelector, MAIN_STORE, DEFAULT_FLAVOR)
90from lp.code.enums import (91from lp.code.enums import (
91 BranchMergeProposalStatus, BranchSubscriptionNotificationLevel,92 BranchMergeProposalStatus, BranchSubscriptionNotificationLevel,
92 BranchType, CodeImportMachineState, CodeImportReviewStatus,93 BranchType, CodeImportMachineState, CodeImportReviewStatus,
@@ -125,8 +126,10 @@
125from lp.registry.interfaces.ssh import ISSHKeySet, SSHKeyType126from lp.registry.interfaces.ssh import ISSHKeySet, SSHKeyType
126from lp.services.worlddata.interfaces.language import ILanguageSet127from lp.services.worlddata.interfaces.language import ILanguageSet
127from lp.buildmaster.interfaces.builder import IBuilderSet128from lp.buildmaster.interfaces.builder import IBuilderSet
129from lp.buildmaster.interfaces.buildfarmjob import BuildFarmJobType
128from lp.soyuz.interfaces.component import IComponentSet130from lp.soyuz.interfaces.component import IComponentSet
129from lp.soyuz.interfaces.packageset import IPackagesetSet131from lp.soyuz.interfaces.packageset import IPackagesetSet
132from lp.soyuz.model.buildqueue import BuildQueue
130from lp.testing import run_with_login, time_counter133from lp.testing import run_with_login, time_counter
131134
132SPACE = ' '135SPACE = ' '
@@ -1615,6 +1618,19 @@
1615 archive=archive,1618 archive=archive,
1616 requester=requester)1619 requester=requester)
16171620
1621 def makeSourcePackageRecipeBuildJob(self, score=9876):
1622 """Create a `SourcePackageRecipeBuildJob` and a `BuildQueue` for
1623 testing."""
1624 recipe_build = self.makeSourcePackageRecipeBuild()
1625 recipe_build_job = recipe_build.makeJob()
1626
1627 store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
1628 bq = BuildQueue(
1629 job=recipe_build_job.job, lastscore=score,
1630 job_type=BuildFarmJobType.RECIPEBRANCHBUILD)
1631 store.add(bq)
1632 return bq
1633
1618 def makePOTemplate(self, productseries=None, distroseries=None,1634 def makePOTemplate(self, productseries=None, distroseries=None,
1619 sourcepackagename=None, owner=None, name=None,1635 sourcepackagename=None, owner=None, name=None,
1620 translation_domain=None, path=None):1636 translation_domain=None, path=None):

Subscribers

People subscribed via source and target branches

to status/vote changes: