Launchpad itself

Merge lp://staging/~al-maisan/launchpad/xx-select-job-506617 into lp://staging/launchpad/db-devel

  1. xx-select-job-506617
  2. Merge into db-devel
Proposed by Muharem Hrnjadovic
Status: Merged
Merged at revision: not available
Proposed branch: lp://staging/~al-maisan/launchpad/xx-select-job-506617
Merge into: lp://staging/launchpad/db-devel
Diff against target: 644 lines (+247/-178)
9 files modified
lib/lp/buildmaster/interfaces/buildfarmjob.py (+39/-0)
lib/lp/buildmaster/model/builder.py (+55/-142)
lib/lp/buildmaster/model/buildfarmjob.py (+13/-2)
lib/lp/soyuz/interfaces/buildpackagejob.py (+2/-2)
lib/lp/soyuz/interfaces/buildqueue.py (+0/-4)
lib/lp/soyuz/model/buildpackagejob.py (+110/-4)
lib/lp/soyuz/model/buildqueue.py (+19/-18)
lib/lp/soyuz/tests/test_buildpackagejob.py (+2/-1)
lib/lp/soyuz/tests/test_buildqueue.py (+7/-5)
To merge this branch: bzr merge lp://staging/~al-maisan/launchpad/xx-select-job-506617
Reviewer Review Type Date Requested Status
Jeroen T. Vermeulen (community) Approve
Review via email: mp+17327@code.staging.launchpad.net
To post a comment you must log in.
Revision history for this message
Muharem Hrnjadovic (al-maisan) wrote :

Hello there!

The branch at hand generalizes the selection of candidate jobs for idle
builders while enabling specific job classes to influence or refine the db
queries used in the process of selection.

Test to build:

    bin/test -vv -t build

No pertinent "make lint" issues.

Revision history for this message
Jeroen T. Vermeulen (jtv) wrote :

We went through this together, resulting in a few very small changes (mainly naming).

I'm approving this branch, but one issue remains open: the new interface, IBuildFarmCandidateJobSelection, plays a role that seems to fit well in the one I added in my branch lp:~jtv/launchpad/bug-500110 called ISpecificBuildFarmJobClass. Consider merging these two interfaces; it may also help prevent conflicts since your branch is going into db-devel whereas mine is for devel.

Jeroen

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'lib/lp/buildmaster/interfaces/buildfarmjob.py'
--- lib/lp/buildmaster/interfaces/buildfarmjob.py 2010-01-08 12:26:22 +0000
+++ lib/lp/buildmaster/interfaces/buildfarmjob.py 2010-01-14 02:33:14 +0000
@@ -9,6 +9,7 @@
99
10__all__ = [10__all__ = [
11 'IBuildFarmJob',11 'IBuildFarmJob',
12 'IBuildFarmCandidateJobSelection',
12 'IBuildFarmJobDispatchEstimation',13 'IBuildFarmJobDispatchEstimation',
13 'BuildFarmJobType',14 'BuildFarmJobType',
14 ]15 ]
@@ -134,3 +135,41 @@
134 the pending jobs of the appropriate type.135 the pending jobs of the appropriate type.
135 """136 """
136137
138
139class IBuildFarmCandidateJobSelection(Interface):
140 """Operations for refining candidate job selection (optional).
141
142 Job type classes that do *not* need to refine candidate job selection may
143 be derived from `BuildFarmJob` which provides a base implementation of
144 this interface.
145 """
146
147 def addCandidateSelectionCriteria(processor, virtualized):
148 """Provide extra clauses that will refine the candidate job selection.
149
150 Return a 2-tuple with extra tables and clauses to be used to
151 narrow down the list of candidate jobs.
152
153 Example:
154 (('Build', 'BuildPackageJob'),
155 "BuildPackageJob.build = Build.id AND ..")
156
157 :param processor: the type of processor that the candidate jobs are
158 expected to run on.
159 :param virtualized: whether the candidate jobs are expected to run on
160 the `processor` natively or inside a virtual machine.
161 :return: an (extra_tables, extra_query) tuple where `extra_tables` is
162 a collection of tables that need to appear in the FROM clause of
163 the combined query for `extra_query` to work.
164 """
165
166 def postprocessCandidate(job, logger):
167 """True if the candidate job is fine and should be dispatched
168 to a builder, False otherwise.
169
170 :param job: The `BuildQueue` instance to be scrutinized.
171 :param logger: The logger to use.
172
173 :return: True if the candidate job should be dispatched
174 to a builder, False otherwise.
175 """
137176
=== modified file 'lib/lp/buildmaster/model/builder.py'
--- lib/lp/buildmaster/model/builder.py 2010-01-13 20:19:43 +0000
+++ lib/lp/buildmaster/model/builder.py 2010-01-14 02:33:14 +0000
@@ -41,24 +41,23 @@
41# These dependencies on soyuz will be removed when getBuildRecords()41# These dependencies on soyuz will be removed when getBuildRecords()
42# is moved, as well as when the generalisation of findBuildCandidate()42# is moved, as well as when the generalisation of findBuildCandidate()
43# is completed.43# is completed.
44from lp.soyuz.model.buildqueue import BuildQueue44from lp.soyuz.model.buildqueue import BuildQueue, specific_job_classes
45from lp.registry.interfaces.person import validate_public_person45from lp.registry.interfaces.person import validate_public_person
46from lp.registry.interfaces.pocket import PackagePublishingPocket
47from canonical.launchpad.helpers import filenameToContentType46from canonical.launchpad.helpers import filenameToContentType
47from lp.services.job.interfaces.job import JobStatus
48from lp.soyuz.interfaces.buildrecords import IHasBuildRecords48from lp.soyuz.interfaces.buildrecords import IHasBuildRecords
49from lp.soyuz.interfaces.distroarchseries import IDistroArchSeriesSet49from lp.soyuz.interfaces.distroarchseries import IDistroArchSeriesSet
50from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet50from canonical.launchpad.interfaces.librarian import ILibraryFileAliasSet
51from canonical.launchpad.webapp.interfaces import NotFoundError51from canonical.launchpad.webapp.interfaces import NotFoundError
52from lp.soyuz.interfaces.archive import ArchivePurpose
53from lp.soyuz.interfaces.build import BuildStatus, IBuildSet52from lp.soyuz.interfaces.build import BuildStatus, IBuildSet
54from lp.buildmaster.interfaces.builder import (53from lp.buildmaster.interfaces.builder import (
55 BuildDaemonError, BuildSlaveFailure, CannotBuild, CannotFetchFile,54 BuildDaemonError, BuildSlaveFailure, CannotBuild, CannotFetchFile,
56 CannotResumeHost, IBuilder, IBuilderSet, ProtocolVersionMismatch)55 CannotResumeHost, IBuilder, IBuilderSet, ProtocolVersionMismatch)
57from lp.soyuz.interfaces.buildqueue import IBuildQueueSet56from lp.soyuz.interfaces.buildqueue import IBuildQueueSet
58from lp.soyuz.interfaces.publishing import (
59 PackagePublishingStatus)
60from lp.soyuz.model.buildpackagejob import BuildPackageJob57from lp.soyuz.model.buildpackagejob import BuildPackageJob
61from canonical.launchpad.webapp import urlappend58from canonical.launchpad.webapp import urlappend
59from canonical.launchpad.webapp.interfaces import (
60 IStoreSelector, MAIN_STORE, DEFAULT_FLAVOR)
62from canonical.lazr.utils import safe_hasattr61from canonical.lazr.utils import safe_hasattr
63from canonical.librarian.utils import copy_and_close62from canonical.librarian.utils import copy_and_close
6463
@@ -409,99 +408,6 @@
409 return False408 return False
410 return True409 return True
411410
412 # XXX cprov 20071116: It should become part of the public
413 # _findBuildCandidate once we start to detect superseded builds
414 # at build creation time.
415 def _findBinaryBuildCandidate(self):
416 """Return the highest priority build candidate for this builder.
417
418 Returns a pending IBuildQueue record queued for this builder
419 processorfamily with the highest lastscore or None if there
420 is no one available.
421 """
422 # If a private build does not yet have its source published then
423 # we temporarily skip it because we want to wait for the publisher
424 # to place the source in the archive, which is where builders
425 # download the source from in the case of private builds (because
426 # it's a secure location).
427 private_statuses = (
428 PackagePublishingStatus.PUBLISHED,
429 PackagePublishingStatus.SUPERSEDED,
430 PackagePublishingStatus.DELETED,
431 )
432 clauses = ["""
433 ((archive.private IS TRUE AND
434 EXISTS (
435 SELECT SourcePackagePublishingHistory.id
436 FROM SourcePackagePublishingHistory
437 WHERE
438 SourcePackagePublishingHistory.distroseries =
439 DistroArchSeries.distroseries AND
440 SourcePackagePublishingHistory.sourcepackagerelease =
441 Build.sourcepackagerelease AND
442 SourcePackagePublishingHistory.archive = Archive.id AND
443 SourcePackagePublishingHistory.status IN %s))
444 OR
445 archive.private IS FALSE) AND
446 buildqueue.job = buildpackagejob.job AND
447 buildpackagejob.build = build.id AND
448 build.distroarchseries = distroarchseries.id AND
449 build.archive = archive.id AND
450 archive.enabled = TRUE AND
451 build.buildstate = %s AND
452 distroarchseries.processorfamily = %s AND
453 buildqueue.builder IS NULL
454 """ % sqlvalues(
455 private_statuses, BuildStatus.NEEDSBUILD, self.processor.family)]
456
457 clauseTables = [
458 'Build', 'BuildPackageJob', 'DistroArchSeries', 'Archive']
459
460 clauses.append("""
461 archive.require_virtualized = %s
462 """ % sqlvalues(self.virtualized))
463
464 # Ensure that if BUILDING builds exist for the same
465 # public ppa archive and architecture and another would not
466 # leave at least 20% of them free, then we don't consider
467 # another as a candidate.
468 #
469 # This clause selects the count of currently building builds on
470 # the arch in question, then adds one to that total before
471 # deriving a percentage of the total available builders on that
472 # arch. It then makes sure that percentage is under 80.
473 #
474 # The extra clause is only used if the number of available
475 # builders is greater than one, or nothing would get dispatched
476 # at all.
477 num_arch_builders = Builder.selectBy(
478 processor=self.processor, manual=False, builderok=True).count()
479 if num_arch_builders > 1:
480 clauses.append("""
481 EXISTS (SELECT true
482 WHERE ((
483 SELECT COUNT(build2.id)
484 FROM Build build2, DistroArchSeries distroarchseries2
485 WHERE
486 build2.archive = build.archive AND
487 archive.purpose = %s AND
488 archive.private IS FALSE AND
489 build2.distroarchseries = distroarchseries2.id AND
490 distroarchseries2.processorfamily = %s AND
491 build2.buildstate = %s) + 1::numeric)
492 *100 / %s
493 < 80)
494 """ % sqlvalues(
495 ArchivePurpose.PPA, self.processor.family,
496 BuildStatus.BUILDING, num_arch_builders))
497
498 query = " AND ".join(clauses)
499 candidate = BuildQueue.selectFirst(
500 query, clauseTables=clauseTables,
501 orderBy=['-buildqueue.lastscore', 'build.id'])
502
503 return candidate
504
505 def _getSlaveScannerLogger(self):411 def _getSlaveScannerLogger(self):
506 """Return the logger instance from buildd-slave-scanner.py."""412 """Return the logger instance from buildd-slave-scanner.py."""
507 # XXX cprov 20071120: Ideally the Launchpad logging system413 # XXX cprov 20071120: Ideally the Launchpad logging system
@@ -515,52 +421,59 @@
515 """Find a candidate job for dispatch to an idle buildd slave.421 """Find a candidate job for dispatch to an idle buildd slave.
516422
517 The pending BuildQueue item with the highest score for this builder423 The pending BuildQueue item with the highest score for this builder
518 ProcessorFamily or None if no candidate is available.424 or None if no candidate is available.
519425
520 For public PPA builds, subsequent builds for a given ppa and426 :return: A candidate job.
521 architecture will not be returned until the current build for
522 the ppa and architecture is finished.
523
524 :return: A binary build candidate job.
525 """427 """
526
527 logger = self._getSlaveScannerLogger()428 logger = self._getSlaveScannerLogger()
528 candidate = self._findBinaryBuildCandidate()429 candidate = None
529430
530 # Mark build records targeted to old source versions as SUPERSEDED431 query_tables = set(('buildqueue', 'job'))
531 # and build records target to SECURITY pocket as FAILEDTOBUILD.432 general_query = """
532 # Builds in those situation should not be built because they will433 SELECT buildqueue.id FROM %%s
533 # be wasting build-time, the former case already has a newer source434 WHERE
534 # and the latter could not be built in DAK.435 buildqueue.job = job.id
535 build_set = getUtility(IBuildSet)436 AND job.status = %s
536 while candidate is not None:437 AND (
537 build = build_set.getByQueueEntry(candidate)438 (buildqueue.processor = %s
538 if build.pocket == PackagePublishingPocket.SECURITY:439 AND buildqueue.virtualized = %s)
539 # We never build anything in the security pocket.440 OR
540 logger.debug(441 (buildqueue.processor IS NULL
541 "Build %s FAILEDTOBUILD, queue item %s REMOVED"442 AND buildqueue.virtualized IS NULL))
542 % (build.id, candidate.id))443 AND buildqueue.builder IS NULL
543 build.buildstate = BuildStatus.FAILEDTOBUILD444 """ % sqlvalues(JobStatus.WAITING, self.processor, self.virtualized)
544 candidate.destroySelf()445 order_clause = " ORDER BY buildqueue.lastscore DESC, buildqueue.id"
545 candidate = self._findBinaryBuildCandidate()446
546 continue447 extra_tables = set()
547448 extra_queries = []
548 publication = build.current_source_publication449 job_classes = specific_job_classes()
549450 for job_class in job_classes.values():
550 if publication is None:451 tables, query = job_class.addCandidateSelectionCriteria(
551 # The build should be superseded if it no longer has a452 self.processor, self.virtualized)
552 # current publishing record.453 if query == '':
553 logger.debug(454 # This job class does not need to refine candidate jobs
554 "Build %s SUPERSEDED, queue item %s REMOVED"455 # further.
555 % (build.id, candidate.id))456 continue
556 build.buildstate = BuildStatus.SUPERSEDED457
557 candidate.destroySelf()458 # Table names are case-insensitive in SQL. All table names are in
558 candidate = self._findBinaryBuildCandidate()459 # lower case in order to avoid duplicates in the FROM clause.
559 continue460 query_tables = query_tables.union(
560461 set(table.lower() for table in tables))
561 return candidate462 extra_queries.append(query)
562463 general_query = general_query % ', '.join(query_tables)
563 # No candidate was found.464 query = ' AND '.join([general_query] + extra_queries) + order_clause
465
466 store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
467 candidate_jobs = store.execute(query).get_all()
468
469 for (candidate_id,) in candidate_jobs:
470 candidate = getUtility(IBuildQueueSet).get(candidate_id)
471 job_class = job_classes[candidate.job_type]
472 candidate_approved = job_class.postprocessCandidate(
473 candidate, logger)
474 if candidate_approved:
475 return candidate
476
564 return None477 return None
565478
566 def _dispatchBuildCandidate(self, candidate):479 def _dispatchBuildCandidate(self, candidate):
567480
=== modified file 'lib/lp/buildmaster/model/buildfarmjob.py'
--- lib/lp/buildmaster/model/buildfarmjob.py 2009-12-24 13:18:16 +0000
+++ lib/lp/buildmaster/model/buildfarmjob.py 2010-01-14 02:33:14 +0000
@@ -5,14 +5,16 @@
5__all__ = ['BuildFarmJob']5__all__ = ['BuildFarmJob']
66
77
8from zope.interface import implements8from zope.interface import classProvides, implements
99
10from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJob10from lp.buildmaster.interfaces.buildfarmjob import (
11 IBuildFarmJob, IBuildFarmCandidateJobSelection)
1112
1213
13class BuildFarmJob:14class BuildFarmJob:
14 """Mix-in class for `IBuildFarmJob` implementations."""15 """Mix-in class for `IBuildFarmJob` implementations."""
15 implements(IBuildFarmJob)16 implements(IBuildFarmJob)
17 classProvides(IBuildFarmCandidateJobSelection)
1618
17 def score(self):19 def score(self):
18 """See `IBuildFarmJob`."""20 """See `IBuildFarmJob`."""
@@ -48,3 +50,12 @@
48 """See `IBuildFarmJob`."""50 """See `IBuildFarmJob`."""
49 return None51 return None
5052
53 @staticmethod
54 def addCandidateSelectionCriteria(processor, virtualized):
55 """See `IBuildFarmCandidateJobSelection`."""
56 return ([], '')
57
58 @staticmethod
59 def postprocessCandidate(job, logger):
60 """See `IBuildFarmCandidateJobSelection`."""
61 return True
5162
=== modified file 'lib/lp/soyuz/interfaces/buildpackagejob.py'
--- lib/lp/soyuz/interfaces/buildpackagejob.py 2009-11-13 20:39:36 +0000
+++ lib/lp/soyuz/interfaces/buildpackagejob.py 2010-01-14 02:33:14 +0000
@@ -11,16 +11,16 @@
11 'IBuildPackageJob',11 'IBuildPackageJob',
12 ]12 ]
1313
14from zope.interface import Interface
14from zope.schema import Int15from zope.schema import Int
1516
16from canonical.launchpad import _17from canonical.launchpad import _
17from lazr.restful.fields import Reference18from lazr.restful.fields import Reference
18from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJob
19from lp.services.job.interfaces.job import IJob19from lp.services.job.interfaces.job import IJob
20from lp.soyuz.interfaces.build import IBuild20from lp.soyuz.interfaces.build import IBuild
2121
2222
23class IBuildPackageJob(IBuildFarmJob):23class IBuildPackageJob(Interface):
24 """A read-only interface for build package jobs."""24 """A read-only interface for build package jobs."""
25 id = Int(title=_('ID'), required=True, readonly=True)25 id = Int(title=_('ID'), required=True, readonly=True)
2626
2727
=== modified file 'lib/lp/soyuz/interfaces/buildqueue.py'
--- lib/lp/soyuz/interfaces/buildqueue.py 2010-01-13 20:36:19 +0000
+++ lib/lp/soyuz/interfaces/buildqueue.py 2010-01-14 02:33:14 +0000
@@ -68,10 +68,6 @@
68 title=_('The builder behavior required to run this job.'),68 title=_('The builder behavior required to run this job.'),
69 required=False, readonly=True)69 required=False, readonly=True)
7070
71 specific_job_classes = Field(
72 title=_('Job classes that may run on the build farm.'),
73 required=True, readonly=True)
74
75 estimated_duration = Timedelta(71 estimated_duration = Timedelta(
76 title=_("Estimated Job Duration"), required=True,72 title=_("Estimated Job Duration"), required=True,
77 description=_("Estimated job duration interval."))73 description=_("Estimated job duration interval."))
7874
=== modified file 'lib/lp/soyuz/model/buildpackagejob.py'
--- lib/lp/soyuz/model/buildpackagejob.py 2010-01-12 10:44:24 +0000
+++ lib/lp/soyuz/model/buildpackagejob.py 2010-01-14 02:33:14 +0000
@@ -11,21 +11,24 @@
11from storm.locals import Int, Reference, Storm11from storm.locals import Int, Reference, Storm
1212
13from zope.interface import classProvides, implements13from zope.interface import classProvides, implements
14from zope.component import getUtility
1415
15from canonical.database.constants import UTC_NOW16from canonical.database.constants import UTC_NOW
16from canonical.database.sqlbase import sqlvalues17from canonical.database.sqlbase import sqlvalues
1718
18from lp.buildmaster.interfaces.buildfarmjob import (19from lp.buildmaster.interfaces.buildfarmjob import (
19 BuildFarmJobType, IBuildFarmJobDispatchEstimation)20 BuildFarmJobType, IBuildFarmJobDispatchEstimation)
21from lp.buildmaster.model.buildfarmjob import BuildFarmJob
20from lp.registry.interfaces.sourcepackage import SourcePackageUrgency22from lp.registry.interfaces.sourcepackage import SourcePackageUrgency
21from lp.registry.interfaces.pocket import PackagePublishingPocket23from lp.registry.interfaces.pocket import PackagePublishingPocket
22from lp.services.job.interfaces.job import JobStatus24from lp.services.job.interfaces.job import JobStatus
23from lp.soyuz.interfaces.archive import ArchivePurpose25from lp.soyuz.interfaces.archive import ArchivePurpose
24from lp.soyuz.interfaces.build import BuildStatus26from lp.soyuz.interfaces.build import BuildStatus, IBuildSet
25from lp.soyuz.interfaces.buildpackagejob import IBuildPackageJob27from lp.soyuz.interfaces.buildpackagejob import IBuildPackageJob
2628from lp.soyuz.interfaces.publishing import PackagePublishingStatus
2729
28class BuildPackageJob(Storm):30
31class BuildPackageJob(Storm, BuildFarmJob):
29 """See `IBuildPackageJob`."""32 """See `IBuildPackageJob`."""
30 implements(IBuildPackageJob)33 implements(IBuildPackageJob)
31 classProvides(IBuildFarmJobDispatchEstimation)34 classProvides(IBuildFarmJobDispatchEstimation)
@@ -209,3 +212,106 @@
209 def virtualized(self):212 def virtualized(self):
210 """See `IBuildFarmJob`."""213 """See `IBuildFarmJob`."""
211 return self.build.is_virtualized214 return self.build.is_virtualized
215
216 @staticmethod
217 def addCandidateSelectionCriteria(processor, virtualized):
218 """See `IBuildFarmCandidateJobSelection`."""
219 # Avoiding circular import.
220 from lp.buildmaster.model.builder import Builder
221
222 private_statuses = (
223 PackagePublishingStatus.PUBLISHED,
224 PackagePublishingStatus.SUPERSEDED,
225 PackagePublishingStatus.DELETED,
226 )
227 extra_tables = [
228 'Archive', 'Build', 'BuildPackageJob', 'DistroArchSeries']
229 extra_clauses = """
230 BuildPackageJob.job = Job.id AND
231 BuildPackageJob.build = Build.id AND
232 Build.distroarchseries = DistroArchSeries.id AND
233 Build.archive = Archive.id AND
234 ((Archive.private IS TRUE AND
235 EXISTS (
236 SELECT SourcePackagePublishingHistory.id
237 FROM SourcePackagePublishingHistory
238 WHERE
239 SourcePackagePublishingHistory.distroseries =
240 DistroArchSeries.distroseries AND
241 SourcePackagePublishingHistory.sourcepackagerelease =
242 Build.sourcepackagerelease AND
243 SourcePackagePublishingHistory.archive = Archive.id AND
244 SourcePackagePublishingHistory.status IN %s))
245 OR
246 archive.private IS FALSE) AND
247 build.buildstate = %s
248 """ % sqlvalues(private_statuses, BuildStatus.NEEDSBUILD)
249
250 # Ensure that if BUILDING builds exist for the same
251 # public ppa archive and architecture and another would not
252 # leave at least 20% of them free, then we don't consider
253 # another as a candidate.
254 #
255 # This clause selects the count of currently building builds on
256 # the arch in question, then adds one to that total before
257 # deriving a percentage of the total available builders on that
258 # arch. It then makes sure that percentage is under 80.
259 #
260 # The extra clause is only used if the number of available
261 # builders is greater than one, or nothing would get dispatched
262 # at all.
263 num_arch_builders = Builder.selectBy(
264 processor=processor, manual=False, builderok=True).count()
265 if num_arch_builders > 1:
266 extra_clauses += """
267 AND EXISTS (SELECT true
268 WHERE ((
269 SELECT COUNT(build2.id)
270 FROM Build build2, DistroArchSeries distroarchseries2
271 WHERE
272 build2.archive = build.archive AND
273 archive.purpose = %s AND
274 archive.private IS FALSE AND
275 build2.distroarchseries = distroarchseries2.id AND
276 distroarchseries2.processorfamily = %s AND
277 build2.buildstate = %s) + 1::numeric)
278 *100 / %s
279 < 80)
280 """ % sqlvalues(
281 ArchivePurpose.PPA, processor.family,
282 BuildStatus.BUILDING, num_arch_builders)
283
284 return(extra_tables, extra_clauses)
285
286 @staticmethod
287 def postprocessCandidate(job, logger):
288 """See `IBuildFarmCandidateJobSelection`."""
289 # Mark build records targeted to old source versions as SUPERSEDED
290 # and build records target to SECURITY pocket as FAILEDTOBUILD.
291 # Builds in those situation should not be built because they will
292 # be wasting build-time, the former case already has a newer source
293 # and the latter could not be built in DAK.
294 build_set = getUtility(IBuildSet)
295
296 build = build_set.getByQueueEntry(job)
297 if build.pocket == PackagePublishingPocket.SECURITY:
298 # We never build anything in the security pocket.
299 logger.debug(
300 "Build %s FAILEDTOBUILD, queue item %s REMOVED"
301 % (build.id, job.id))
302 build.buildstate = BuildStatus.FAILEDTOBUILD
303 job.destroySelf()
304 return False
305
306 publication = build.current_source_publication
307 if publication is None:
308 # The build should be superseded if it no longer has a
309 # current publishing record.
310 logger.debug(
311 "Build %s SUPERSEDED, queue item %s REMOVED"
312 % (build.id, job.id))
313 build.buildstate = BuildStatus.SUPERSEDED
314 job.destroySelf()
315 return False
316
317 return True
212318
=== modified file 'lib/lp/soyuz/model/buildqueue.py'
--- lib/lp/soyuz/model/buildqueue.py 2010-01-13 11:11:40 +0000
+++ lib/lp/soyuz/model/buildqueue.py 2010-01-14 02:33:14 +0000
@@ -7,7 +7,8 @@
77
8__all__ = [8__all__ = [
9 'BuildQueue',9 'BuildQueue',
10 'BuildQueueSet'10 'BuildQueueSet',
11 'specific_job_classes',
11 ]12 ]
1213
13import logging14import logging
@@ -38,6 +39,22 @@
38 IStoreSelector, MAIN_STORE, DEFAULT_FLAVOR)39 IStoreSelector, MAIN_STORE, DEFAULT_FLAVOR)
3940
4041
42def specific_job_classes():
43 """Job classes that may run on the build farm."""
44 job_classes = dict()
45 # Get all components that implement the `IBuildFarmJob` interface.
46 components = getSiteManager()
47 implementations = sorted(components.getUtilitiesFor(IBuildFarmJob))
48 # The above yields a collection of 2-tuples where the first element
49 # is the name of the `BuildFarmJobType` enum and the second element
50 # is the implementing class respectively.
51 for job_enum_name, job_class in implementations:
52 job_enum = getattr(BuildFarmJobType, job_enum_name)
53 job_classes[job_enum] = job_class
54
55 return job_classes
56
57
41class BuildQueue(SQLBase):58class BuildQueue(SQLBase):
42 implements(IBuildQueue)59 implements(IBuildQueue)
43 _table = "BuildQueue"60 _table = "BuildQueue"
@@ -62,25 +79,9 @@
62 return IBuildFarmJobBehavior(self.specific_job)79 return IBuildFarmJobBehavior(self.specific_job)
6380
64 @property81 @property
65 def specific_job_classes(self):
66 """See `IBuildQueue`."""
67 job_classes = dict()
68 # Get all components that implement the `IBuildFarmJob` interface.
69 components = getSiteManager()
70 implementations = sorted(components.getUtilitiesFor(IBuildFarmJob))
71 # The above yields a collection of 2-tuples where the first element
72 # is the name of the `BuildFarmJobType` enum and the second element
73 # is the implementing class respectively.
74 for job_enum_name, job_class in implementations:
75 job_enum = getattr(BuildFarmJobType, job_enum_name)
76 job_classes[job_enum] = job_class
77
78 return job_classes
79
80 @property
81 def specific_job(self):82 def specific_job(self):
82 """See `IBuildQueue`."""83 """See `IBuildQueue`."""
83 specific_class = self.specific_job_classes[self.job_type]84 specific_class = specific_job_classes()[self.job_type]
84 store = Store.of(self)85 store = Store.of(self)
85 result_set = store.find(86 result_set = store.find(
86 specific_class, specific_class.job == self.job)87 specific_class, specific_class.job == self.job)
8788
=== modified file 'lib/lp/soyuz/tests/test_buildpackagejob.py'
--- lib/lp/soyuz/tests/test_buildpackagejob.py 2010-01-11 23:43:59 +0000
+++ lib/lp/soyuz/tests/test_buildpackagejob.py 2010-01-14 02:33:14 +0000
@@ -12,11 +12,11 @@
12from canonical.launchpad.webapp.testing import verifyObject12from canonical.launchpad.webapp.testing import verifyObject
13from canonical.testing import LaunchpadZopelessLayer13from canonical.testing import LaunchpadZopelessLayer
1414
15from lp.buildmaster.interfaces.builder import IBuilderSet
15from lp.buildmaster.interfaces.buildfarmjob import (16from lp.buildmaster.interfaces.buildfarmjob import (
16 IBuildFarmJobDispatchEstimation)17 IBuildFarmJobDispatchEstimation)
17from lp.soyuz.interfaces.archive import ArchivePurpose18from lp.soyuz.interfaces.archive import ArchivePurpose
18from lp.soyuz.interfaces.build import BuildStatus19from lp.soyuz.interfaces.build import BuildStatus
19from lp.buildmaster.interfaces.builder import IBuilderSet
20from lp.soyuz.interfaces.publishing import PackagePublishingStatus20from lp.soyuz.interfaces.publishing import PackagePublishingStatus
21from lp.soyuz.model.build import Build21from lp.soyuz.model.build import Build
22from lp.soyuz.model.buildpackagejob import BuildPackageJob22from lp.soyuz.model.buildpackagejob import BuildPackageJob
@@ -24,6 +24,7 @@
24from lp.soyuz.tests.test_publishing import SoyuzTestPublisher24from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
25from lp.testing import TestCaseWithFactory25from lp.testing import TestCaseWithFactory
2626
27
27def find_job(test, name, processor='386'):28def find_job(test, name, processor='386'):
28 """Find build and queue instance for the given source and processor."""29 """Find build and queue instance for the given source and processor."""
29 for build in test.builds:30 for build in test.builds:
3031
=== modified file 'lib/lp/soyuz/tests/test_buildqueue.py'
--- lib/lp/soyuz/tests/test_buildqueue.py 2010-01-13 11:11:40 +0000
+++ lib/lp/soyuz/tests/test_buildqueue.py 2010-01-14 02:33:14 +0000
@@ -13,10 +13,12 @@
13 IStoreSelector, MAIN_STORE, DEFAULT_FLAVOR)13 IStoreSelector, MAIN_STORE, DEFAULT_FLAVOR)
14from canonical.testing import LaunchpadZopelessLayer14from canonical.testing import LaunchpadZopelessLayer
1515
16from lp.buildmaster.interfaces.builder import IBuilderSet
16from lp.buildmaster.interfaces.buildfarmjob import BuildFarmJobType17from lp.buildmaster.interfaces.buildfarmjob import BuildFarmJobType
18from lp.buildmaster.model.builder import specific_job_classes
19from lp.buildmaster.model.buildfarmjob import BuildFarmJob
17from lp.soyuz.interfaces.archive import ArchivePurpose20from lp.soyuz.interfaces.archive import ArchivePurpose
18from lp.soyuz.interfaces.build import BuildStatus21from lp.soyuz.interfaces.build import BuildStatus
19from lp.buildmaster.interfaces.builder import IBuilderSet
20from lp.soyuz.model.processor import ProcessorFamilySet22from lp.soyuz.model.processor import ProcessorFamilySet
21from lp.soyuz.interfaces.publishing import PackagePublishingStatus23from lp.soyuz.interfaces.publishing import PackagePublishingStatus
22from lp.soyuz.model.build import Build24from lp.soyuz.model.build import Build
@@ -705,7 +707,7 @@
705707
706 # The class registered for 'PACKAGEBUILD' is `BuildPackageJob`.708 # The class registered for 'PACKAGEBUILD' is `BuildPackageJob`.
707 self.assertEqual(709 self.assertEqual(
708 bq.specific_job_classes[BuildFarmJobType.PACKAGEBUILD],710 specific_job_classes()[BuildFarmJobType.PACKAGEBUILD],
709 BuildPackageJob,711 BuildPackageJob,
710 "The class registered for 'PACKAGEBUILD' is `BuildPackageJob`")712 "The class registered for 'PACKAGEBUILD' is `BuildPackageJob`")
711713
@@ -721,14 +723,14 @@
721 """Other job type classes are picked up as well."""723 """Other job type classes are picked up as well."""
722 from zope import component724 from zope import component
723 from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJob725 from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJob
724 class FakeBranchBuild:726 class FakeBranchBuild(BuildFarmJob):
725 pass727 pass
726728
727 _build, bq = find_job(self, 'gedit')729 _build, bq = find_job(self, 'gedit')
728 # First make sure that we don't have a job type class registered for730 # First make sure that we don't have a job type class registered for
729 # 'BRANCHBUILD' yet.731 # 'BRANCHBUILD' yet.
730 self.assertTrue(732 self.assertTrue(
731 bq.specific_job_classes.get(BuildFarmJobType.BRANCHBUILD) is None)733 specific_job_classes().get(BuildFarmJobType.BRANCHBUILD) is None)
732734
733 # Pretend that our `FakeBranchBuild` class implements the735 # Pretend that our `FakeBranchBuild` class implements the
734 # `IBuildFarmJob` interface.736 # `IBuildFarmJob` interface.
@@ -738,7 +740,7 @@
738 # Now we should see the `FakeBranchBuild` class "registered" in the740 # Now we should see the `FakeBranchBuild` class "registered" in the
739 # `specific_job_classes` dictionary under the 'BRANCHBUILD' key.741 # `specific_job_classes` dictionary under the 'BRANCHBUILD' key.
740 self.assertEqual(742 self.assertEqual(
741 bq.specific_job_classes[BuildFarmJobType.BRANCHBUILD],743 specific_job_classes()[BuildFarmJobType.BRANCHBUILD],
742 FakeBranchBuild)744 FakeBranchBuild)
743745
744746

Subscribers

People subscribed via source and target branches

to status/vote changes: