Merge lp://staging/~al-maisan/launchpad/empty-table-507782 into lp://staging/launchpad/db-devel
- empty-table-507782
- Merge into db-devel
Proposed by
Muharem Hrnjadovic
Status: | Merged |
---|---|
Merged at revision: | not available |
Proposed branch: | lp://staging/~al-maisan/launchpad/empty-table-507782 |
Merge into: | lp://staging/launchpad/db-devel |
Diff against target: |
359 lines (+120/-38) 8 files modified
lib/lp/buildmaster/interfaces/buildfarmjob.py (+17/-11) lib/lp/buildmaster/model/builder.py (+3/-9) lib/lp/buildmaster/model/buildfarmjob.py (+1/-1) lib/lp/buildmaster/tests/test_builder.py (+66/-6) lib/lp/soyuz/configure.zcml (+3/-0) lib/lp/soyuz/model/buildpackagejob.py (+5/-5) lib/lp/soyuz/model/sourcepackagerecipebuild.py (+8/-5) lib/lp/testing/factory.py (+17/-1) |
To merge this branch: | bzr merge lp://staging/~al-maisan/launchpad/empty-table-507782 |
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Canonical Launchpad Engineering | Pending | ||
Review via email: mp+17566@code.staging.launchpad.net |
Commit message
Description of the change
To post a comment you must log in.
Revision history for this message
Muharem Hrnjadovic (al-maisan) wrote : | # |
Preview Diff
Failed to fetch available diffs.
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'lib/lp/buildmaster/interfaces/buildfarmjob.py' |
2 | --- lib/lp/buildmaster/interfaces/buildfarmjob.py 2010-01-18 00:39:57 +0000 |
3 | +++ lib/lp/buildmaster/interfaces/buildfarmjob.py 2010-01-18 01:45:24 +0000 |
4 | @@ -165,22 +165,28 @@ |
5 | """ |
6 | |
7 | def addCandidateSelectionCriteria(processor, virtualized): |
8 | - """Provide extra clauses that will refine the candidate job selection. |
9 | - |
10 | - Return a 2-tuple with extra tables and clauses to be used to |
11 | - narrow down the list of candidate jobs. |
12 | - |
13 | - Example: |
14 | - (('Build', 'BuildPackageJob'), |
15 | - "BuildPackageJob.build = Build.id AND ..") |
16 | + """Provide a sub-query to refine the candidate job selection. |
17 | + |
18 | + Return a sub-query to narrow down the list of candidate jobs. |
19 | + The sub-query will become part of an "outer query" and is free to |
20 | + refer to the `BuildQueue` and `Job` tables already utilized in the |
21 | + latter. |
22 | + |
23 | + Example (please see the `BuildPackageJob` implementation for a |
24 | + complete example): |
25 | + |
26 | + SELECT TRUE |
27 | + FROM Archive, Build, BuildPackageJob, DistroArchSeries |
28 | + WHERE |
29 | + BuildPackageJob.job = Job.id AND |
30 | + .. |
31 | |
32 | :param processor: the type of processor that the candidate jobs are |
33 | expected to run on. |
34 | :param virtualized: whether the candidate jobs are expected to run on |
35 | the `processor` natively or inside a virtual machine. |
36 | - :return: an (extra_tables, extra_query) tuple where `extra_tables` is |
37 | - a collection of tables that need to appear in the FROM clause of |
38 | - the combined query for `extra_query` to work. |
39 | + :return: a string containing a sub-query that narrows down the list of |
40 | + candidate jobs. |
41 | """ |
42 | |
43 | def postprocessCandidate(job, logger): |
44 | |
45 | === modified file 'lib/lp/buildmaster/model/builder.py' |
46 | --- lib/lp/buildmaster/model/builder.py 2010-01-16 05:54:16 +0000 |
47 | +++ lib/lp/buildmaster/model/builder.py 2010-01-18 01:45:24 +0000 |
48 | @@ -437,7 +437,7 @@ |
49 | def qualify_subquery(job_type, sub_query): |
50 | """Put the sub-query into a job type context.""" |
51 | qualified_query = """ |
52 | - ((BuildQueue.job_type != %s) OR (%%s)) |
53 | + ((BuildQueue.job_type != %s) OR EXISTS(%%s)) |
54 | """ % sqlvalues(job_type) |
55 | qualified_query %= sub_query |
56 | return qualified_query |
57 | @@ -445,9 +445,8 @@ |
58 | logger = self._getSlaveScannerLogger() |
59 | candidate = None |
60 | |
61 | - query_tables = set(('buildqueue', 'job')) |
62 | general_query = """ |
63 | - SELECT buildqueue.id FROM %%s |
64 | + SELECT buildqueue.id FROM buildqueue, job |
65 | WHERE |
66 | buildqueue.job = job.id |
67 | AND job.status = %s |
68 | @@ -474,20 +473,15 @@ |
69 | extra_queries = [] |
70 | job_classes = specific_job_classes() |
71 | for job_type, job_class in job_classes.iteritems(): |
72 | - tables, query = job_class.addCandidateSelectionCriteria( |
73 | + query = job_class.addCandidateSelectionCriteria( |
74 | self.processor, self.virtualized) |
75 | if query == '': |
76 | # This job class does not need to refine candidate jobs |
77 | # further. |
78 | continue |
79 | |
80 | - # Table names are case-insensitive in SQL. All table names are in |
81 | - # lower case in order to avoid duplicates in the FROM clause. |
82 | - query_tables = query_tables.union( |
83 | - set(table.lower() for table in tables)) |
84 | # The sub-query should only apply to jobs of the right type. |
85 | extra_queries.append(qualify_subquery(job_type, query)) |
86 | - general_query = general_query % ', '.join(query_tables) |
87 | query = ' AND '.join([general_query] + extra_queries) + order_clause |
88 | |
89 | store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR) |
90 | |
91 | === modified file 'lib/lp/buildmaster/model/buildfarmjob.py' |
92 | --- lib/lp/buildmaster/model/buildfarmjob.py 2010-01-16 07:52:28 +0000 |
93 | +++ lib/lp/buildmaster/model/buildfarmjob.py 2010-01-18 01:45:24 +0000 |
94 | @@ -62,7 +62,7 @@ |
95 | @staticmethod |
96 | def addCandidateSelectionCriteria(processor, virtualized): |
97 | """See `IBuildFarmCandidateJobSelection`.""" |
98 | - return ([], '') |
99 | + return ('') |
100 | |
101 | @classmethod |
102 | def getByJob(cls, job): |
103 | |
104 | === modified file 'lib/lp/buildmaster/tests/test_builder.py' |
105 | --- lib/lp/buildmaster/tests/test_builder.py 2010-01-12 20:26:34 +0000 |
106 | +++ lib/lp/buildmaster/tests/test_builder.py 2010-01-18 01:45:24 +0000 |
107 | @@ -8,16 +8,22 @@ |
108 | from zope.component import getUtility |
109 | from zope.security.proxy import removeSecurityProxy |
110 | |
111 | +from canonical.launchpad.webapp.interfaces import ( |
112 | + IStoreSelector, MAIN_STORE, DEFAULT_FLAVOR) |
113 | from canonical.testing import LaunchpadZopelessLayer |
114 | +from lp.buildmaster.interfaces.builder import IBuilderSet |
115 | +from lp.buildmaster.interfaces.buildfarmjob import BuildFarmJobType |
116 | from lp.buildmaster.interfaces.buildfarmjobbehavior import ( |
117 | IBuildFarmJobBehavior) |
118 | from lp.buildmaster.model.buildfarmjobbehavior import IdleBuildBehavior |
119 | from lp.soyuz.interfaces.archive import ArchivePurpose |
120 | from lp.soyuz.interfaces.build import BuildStatus, IBuildSet |
121 | -from lp.buildmaster.interfaces.builder import IBuilderSet |
122 | from lp.soyuz.interfaces.publishing import PackagePublishingStatus |
123 | +from lp.soyuz.interfaces.sourcepackagerecipebuild import ( |
124 | + ISourcePackageRecipeBuildSource) |
125 | from lp.soyuz.model.binarypackagebuildbehavior import ( |
126 | BinaryPackageBuildBehavior) |
127 | +from lp.soyuz.model.buildqueue import BuildQueue |
128 | from lp.soyuz.tests.test_publishing import SoyuzTestPublisher |
129 | from lp.testing import TestCaseWithFactory |
130 | |
131 | @@ -194,15 +200,15 @@ |
132 | super(TestFindBuildCandidateDistroArchive, self).setUp() |
133 | # Create a primary archive and publish some builds for the |
134 | # queue. |
135 | - non_ppa = self.factory.makeArchive( |
136 | + self.non_ppa = self.factory.makeArchive( |
137 | name="primary", purpose=ArchivePurpose.PRIMARY) |
138 | |
139 | - gedit_build = self.publisher.getPubSource( |
140 | + self.gedit_build = self.publisher.getPubSource( |
141 | sourcename="gedit", status=PackagePublishingStatus.PUBLISHED, |
142 | - archive=non_ppa).createMissingBuilds()[0] |
143 | - firefox_build = self.publisher.getPubSource( |
144 | + archive=self.non_ppa).createMissingBuilds()[0] |
145 | + self.firefox_build = self.publisher.getPubSource( |
146 | sourcename="firefox", status=PackagePublishingStatus.PUBLISHED, |
147 | - archive=non_ppa).createMissingBuilds()[0] |
148 | + archive=self.non_ppa).createMissingBuilds()[0] |
149 | |
150 | def test_findBuildCandidate_for_non_ppa(self): |
151 | # Normal archives are not restricted to serial builds per |
152 | @@ -224,6 +230,60 @@ |
153 | self.failUnlessEqual('primary', build.archive.name) |
154 | self.failUnlessEqual('firefox', build.sourcepackagerelease.name) |
155 | |
156 | + def test_findBuildCandidate_for_recipe_build(self): |
157 | + # Recipe builds with a higher score are selected first. |
158 | + # This test is run in a context with mixed recipe and binary builds. |
159 | + |
160 | + self.assertIsNot(self.frog_builder.processor, None) |
161 | + self.assertEqual(self.frog_builder.virtualized, True) |
162 | + |
163 | + self.assertEqual(self.gedit_build.buildqueue_record.lastscore, 2505) |
164 | + self.assertEqual(self.firefox_build.buildqueue_record.lastscore, 2505) |
165 | + |
166 | + recipe_build_job = self.factory.makeSourcePackageRecipeBuildJob(9999) |
167 | + |
168 | + self.assertEqual(recipe_build_job.lastscore, 9999) |
169 | + |
170 | + next_job = removeSecurityProxy( |
171 | + self.frog_builder)._findBuildCandidate() |
172 | + |
173 | + self.failUnlessEqual(recipe_build_job, next_job) |
174 | + |
175 | + |
176 | +class TestFindRecipeBuildCandidates(TestFindBuildCandidateBase): |
177 | + # These tests operate in a "recipe builds only" setting. |
178 | + # Please see also bug #507782. |
179 | + |
180 | + def clearBuildQueue(self): |
181 | + """Delete all `BuildQueue`, XXXJOb and `Job` instances.""" |
182 | + store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR) |
183 | + for bq in store.find(BuildQueue): |
184 | + bq.destroySelf() |
185 | + |
186 | + def setUp(self): |
187 | + """Publish some builds for the test archive.""" |
188 | + super(TestFindRecipeBuildCandidates, self).setUp() |
189 | + # Create a primary archive and publish some builds for the |
190 | + # queue. |
191 | + self.non_ppa = self.factory.makeArchive( |
192 | + name="primary", purpose=ArchivePurpose.PRIMARY) |
193 | + |
194 | + self.clearBuildQueue() |
195 | + self.bq1 = self.factory.makeSourcePackageRecipeBuildJob(3333) |
196 | + self.bq2 = self.factory.makeSourcePackageRecipeBuildJob(4333) |
197 | + |
198 | + def test_findBuildCandidate_with_highest_score(self): |
199 | + # The recipe build with the highest score is selected first. |
200 | + # This test is run in a "recipe builds only" context. |
201 | + |
202 | + self.assertIsNot(self.frog_builder.processor, None) |
203 | + self.assertEqual(self.frog_builder.virtualized, True) |
204 | + |
205 | + next_job = removeSecurityProxy( |
206 | + self.frog_builder)._findBuildCandidate() |
207 | + |
208 | + self.failUnlessEqual(self.bq2, next_job) |
209 | + |
210 | |
211 | class TestCurrentBuildBehavior(TestCaseWithFactory): |
212 | """This test ensures the get/set behavior of IBuilder's |
213 | |
214 | === modified file 'lib/lp/soyuz/configure.zcml' |
215 | --- lib/lp/soyuz/configure.zcml 2010-01-17 05:01:30 +0000 |
216 | +++ lib/lp/soyuz/configure.zcml 2010-01-18 01:45:24 +0000 |
217 | @@ -962,6 +962,9 @@ |
218 | class="lp.soyuz.model.sourcepackagerecipebuild.SourcePackageRecipeBuildJob"> |
219 | <allow interface="lp.soyuz.interfaces.sourcepackagerecipebuild.ISourcePackageRecipeBuildJob"/> |
220 | </class> |
221 | + <utility component="lp.soyuz.model.sourcepackagerecipebuild.SourcePackageRecipeBuildJob" |
222 | + name="RECIPEBRANCHBUILD" |
223 | + provides="lp.buildmaster.interfaces.buildfarmjob.IBuildFarmJob"/> |
224 | |
225 | <securedutility |
226 | component="lp.soyuz.model.sourcepackagerecipebuild.SourcePackageRecipeBuildJob" |
227 | |
228 | === modified file 'lib/lp/soyuz/model/buildpackagejob.py' |
229 | --- lib/lp/soyuz/model/buildpackagejob.py 2010-01-16 09:45:58 +0000 |
230 | +++ lib/lp/soyuz/model/buildpackagejob.py 2010-01-18 01:45:24 +0000 |
231 | @@ -230,9 +230,9 @@ |
232 | PackagePublishingStatus.SUPERSEDED, |
233 | PackagePublishingStatus.DELETED, |
234 | ) |
235 | - extra_tables = [ |
236 | - 'Archive', 'Build', 'BuildPackageJob', 'DistroArchSeries'] |
237 | - extra_clauses = """ |
238 | + sub_query = """ |
239 | + SELECT TRUE FROM Archive, Build, BuildPackageJob, DistroArchSeries |
240 | + WHERE |
241 | BuildPackageJob.job = Job.id AND |
242 | BuildPackageJob.build = Build.id AND |
243 | Build.distroarchseries = DistroArchSeries.id AND |
244 | @@ -269,7 +269,7 @@ |
245 | num_arch_builders = Builder.selectBy( |
246 | processor=processor, manual=False, builderok=True).count() |
247 | if num_arch_builders > 1: |
248 | - extra_clauses += """ |
249 | + sub_query += """ |
250 | AND EXISTS (SELECT true |
251 | WHERE (( |
252 | SELECT COUNT(build2.id) |
253 | @@ -287,7 +287,7 @@ |
254 | ArchivePurpose.PPA, processor.family, |
255 | BuildStatus.BUILDING, num_arch_builders) |
256 | |
257 | - return(extra_tables, extra_clauses) |
258 | + return sub_query |
259 | |
260 | @staticmethod |
261 | def postprocessCandidate(job, logger): |
262 | |
263 | === modified file 'lib/lp/soyuz/model/sourcepackagerecipebuild.py' |
264 | --- lib/lp/soyuz/model/sourcepackagerecipebuild.py 2010-01-18 00:39:57 +0000 |
265 | +++ lib/lp/soyuz/model/sourcepackagerecipebuild.py 2010-01-18 01:45:24 +0000 |
266 | @@ -19,8 +19,9 @@ |
267 | from zope.component import getUtility |
268 | from zope.interface import classProvides, implements |
269 | |
270 | +from lp.buildmaster.model.buildfarmjob import BuildFarmJob |
271 | +from lp.registry.interfaces.pocket import PackagePublishingPocket |
272 | from lp.services.job.model.job import Job |
273 | -from lp.registry.interfaces.pocket import PackagePublishingPocket |
274 | from lp.soyuz.adapters.archivedependencies import ( |
275 | default_component_dependency_name,) |
276 | from lp.soyuz.interfaces.build import BuildStatus |
277 | @@ -65,7 +66,8 @@ |
278 | distroseries = Reference(distroseries_id, 'DistroSeries.id') |
279 | |
280 | sourcepackagename_id = Int(name='sourcepackagename', allow_none=True) |
281 | - sourcepackagename = Reference(sourcepackagename_id, 'SourcePackageName.id') |
282 | + sourcepackagename = Reference( |
283 | + sourcepackagename_id, 'SourcePackageName.id') |
284 | |
285 | @property |
286 | def pocket(self): |
287 | @@ -101,7 +103,8 @@ |
288 | self.sourcepackagename = sourcepackagename |
289 | |
290 | @classmethod |
291 | - def new(cls, sourcepackage, recipe, requester, archive, date_created=None): |
292 | + def new( |
293 | + cls, sourcepackage, recipe, requester, archive, date_created=None): |
294 | """See `ISourcePackageRecipeBuildSource`.""" |
295 | store = IMasterStore(SourcePackageRecipeBuild) |
296 | if date_created is None: |
297 | @@ -126,7 +129,7 @@ |
298 | return specific_job |
299 | |
300 | |
301 | -class SourcePackageRecipeBuildJob(Storm): |
302 | +class SourcePackageRecipeBuildJob(BuildFarmJob, Storm): |
303 | |
304 | classProvides(ISourcePackageRecipeBuildJobSource) |
305 | implements(ISourcePackageRecipeBuildJob) |
306 | @@ -143,7 +146,7 @@ |
307 | source_package_build_id, 'SourcePackageRecipeBuild.id') |
308 | |
309 | processor = None |
310 | - virtualized = False |
311 | + virtualized = True |
312 | |
313 | def __init__(self, build, job): |
314 | super(SourcePackageRecipeBuildJob, self).__init__() |
315 | |
316 | === modified file 'lib/lp/testing/factory.py' |
317 | --- lib/lp/testing/factory.py 2010-01-17 23:32:21 +0000 |
318 | +++ lib/lp/testing/factory.py 2010-01-18 01:45:24 +0000 |
319 | @@ -86,7 +86,8 @@ |
320 | from lp.services.mail.signedmessage import SignedMessage |
321 | from lp.services.worlddata.interfaces.country import ICountrySet |
322 | from canonical.launchpad.webapp.dbpolicy import MasterDatabasePolicy |
323 | -from canonical.launchpad.webapp.interfaces import IStoreSelector |
324 | +from canonical.launchpad.webapp.interfaces import ( |
325 | + IStoreSelector, MAIN_STORE, DEFAULT_FLAVOR) |
326 | from lp.code.enums import ( |
327 | BranchMergeProposalStatus, BranchSubscriptionNotificationLevel, |
328 | BranchType, CodeImportMachineState, CodeImportReviewStatus, |
329 | @@ -125,8 +126,10 @@ |
330 | from lp.registry.interfaces.ssh import ISSHKeySet, SSHKeyType |
331 | from lp.services.worlddata.interfaces.language import ILanguageSet |
332 | from lp.buildmaster.interfaces.builder import IBuilderSet |
333 | +from lp.buildmaster.interfaces.buildfarmjob import BuildFarmJobType |
334 | from lp.soyuz.interfaces.component import IComponentSet |
335 | from lp.soyuz.interfaces.packageset import IPackagesetSet |
336 | +from lp.soyuz.model.buildqueue import BuildQueue |
337 | from lp.testing import run_with_login, time_counter |
338 | |
339 | SPACE = ' ' |
340 | @@ -1615,6 +1618,19 @@ |
341 | archive=archive, |
342 | requester=requester) |
343 | |
344 | + def makeSourcePackageRecipeBuildJob(self, score=9876): |
345 | + """Create a `SourcePackageRecipeBuildJob` and a `BuildQueue` for |
346 | + testing.""" |
347 | + recipe_build = self.makeSourcePackageRecipeBuild() |
348 | + recipe_build_job = recipe_build.makeJob() |
349 | + |
350 | + store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR) |
351 | + bq = BuildQueue( |
352 | + job=recipe_build_job.job, lastscore=score, |
353 | + job_type=BuildFarmJobType.RECIPEBRANCHBUILD) |
354 | + store.add(bq) |
355 | + return bq |
356 | + |
357 | def makePOTemplate(self, productseries=None, distroseries=None, |
358 | sourcepackagename=None, owner=None, name=None, |
359 | translation_domain=None, path=None): |
This branch fixes (first and foremost) the following bug: source package recipe build jobs are not seen in the absence of binary build jobs. This makes the candidate job selection fail.
Test to run:
bin/test -vv build