Launchpad itself

Merge lp://staging/~al-maisan/launchpad/builder-data-502841 into lp://staging/launchpad

  1. builder-data-502841
  2. Merge into devel
Proposed by Muharem Hrnjadovic
Status: Merged
Merged at revision: not available
Proposed branch: lp://staging/~al-maisan/launchpad/builder-data-502841
Merge into: lp://staging/launchpad
Diff against target: 403 lines (+388/-0)
2 files modified
lib/lp/soyuz/model/buildqueue.py (+56/-0)
lib/lp/soyuz/tests/test_buildqueue.py (+332/-0)
To merge this branch: bzr merge lp://staging/~al-maisan/launchpad/builder-data-502841
Reviewer Review Type Date Requested Status
Eleanor Berger (community) Approve
Review via email: mp+16774@code.staging.launchpad.net
To post a comment you must log in.
Revision history for this message
Muharem Hrnjadovic (al-maisan) wrote :

Hello there!

This is another piece required to get us to the point where we can estimate
job dispatch times in the Soyuz build farm irrespective of job type.

The code added

  - retrieves data about the builders that are available in the system for
    the various processor architectures and virtualization settings.
  - check how many builders are free/available to take on a job for a given
    build (processor and virtualization setting).

Tests to run:

    bin/test -vv -t TestBuilderData

No "make lint" errors or warnings.

Revision history for this message
Eleanor Berger (intellectronica) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/soyuz/model/buildqueue.py'
2--- lib/lp/soyuz/model/buildqueue.py 2009-12-10 20:06:11 +0000
3+++ lib/lp/soyuz/model/buildqueue.py 2010-01-04 14:33:16 +0000
4@@ -156,6 +156,62 @@
5 """See `IBuildQueue`."""
6 self.job.date_started = timestamp
7
8+ def _getBuilderData(self):
9+ """How many working builders are there, how are they configured?"""
10+ # Please note: this method will send only one request to the database.
11+
12+ store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
13+ my_processor = self.specific_job.processor
14+ my_virtualized = self.specific_job.virtualized
15+
16+ # We need to know the total number of builders as well as the
17+ # number of builders that can run the job of interest (JOI).
18+ # If the JOI is processor independent these builder counts will
19+ # have the same value.
20+ builder_data = """
21+ SELECT processor, virtualized, COUNT(id) FROM builder
22+ WHERE builderok = TRUE AND manual = FALSE
23+ GROUP BY processor, virtualized;
24+ """
25+ results = store.execute(builder_data).get_all()
26+
27+ builder_stats = dict()
28+ builders_in_total = builders_for_job = 0
29+ for processor, virtualized, count in results:
30+ if my_processor is not None:
31+ if (my_processor.id == processor and
32+ my_virtualized == virtualized):
33+ # The job on hand can only run on builders with a
34+ # particular processor/virtualization combination and
35+ # this is how many of these we have.
36+ builders_for_job = count
37+ builders_in_total += count
38+ builder_stats[(processor, virtualized)] = count
39+ if my_processor is None:
40+ # The job of interest (JOI) is processor independent.
41+ builders_for_job = builders_in_total
42+
43+ return (builders_in_total, builders_for_job, builder_stats)
44+
45+ def _freeBuildersCount(self, processor, virtualized):
46+ """How many builders capable of running jobs for the given processor
47+ and virtualization combination are idle/free at present?"""
48+ query = """
49+ SELECT COUNT(id) FROM builder
50+ WHERE
51+ builderok = TRUE AND manual = FALSE
52+ AND id NOT IN (
53+ SELECT builder FROM BuildQueue WHERE builder IS NOT NULL)
54+ """
55+ if processor is not None:
56+ query += """
57+ AND processor = %s AND virtualized = %s
58+ """ % sqlvalues(processor, virtualized)
59+ store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
60+ result_set = store.execute(query)
61+ free_builders = result_set.get_one()[0]
62+ return free_builders
63+
64
65 class BuildQueueSet(object):
66 """Utility to deal with BuildQueue content class."""
67
68=== added file 'lib/lp/soyuz/tests/test_buildqueue.py'
69--- lib/lp/soyuz/tests/test_buildqueue.py 1970-01-01 00:00:00 +0000
70+++ lib/lp/soyuz/tests/test_buildqueue.py 2010-01-04 14:33:16 +0000
71@@ -0,0 +1,332 @@
72+# Copyright 2009 Canonical Ltd. This software is licensed under the
73+# GNU Affero General Public License version 3 (see the file LICENSE).
74+# pylint: disable-msg=C0324
75+
76+"""Test BuildQueue features."""
77+
78+from datetime import timedelta
79+
80+from zope.component import getUtility
81+
82+from canonical.launchpad.webapp.interfaces import (
83+ IStoreSelector, MAIN_STORE, DEFAULT_FLAVOR)
84+from canonical.testing import LaunchpadZopelessLayer
85+
86+from lp.soyuz.interfaces.archive import ArchivePurpose
87+from lp.soyuz.interfaces.build import BuildStatus
88+from lp.soyuz.interfaces.builder import IBuilderSet
89+from lp.soyuz.model.processor import ProcessorFamilySet
90+from lp.soyuz.interfaces.publishing import PackagePublishingStatus
91+from lp.soyuz.model.build import Build
92+from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
93+from lp.testing import TestCaseWithFactory
94+
95+
96+def find_job(test, name, processor='386'):
97+ """Find build and queue instance for the given source and processor."""
98+ for build in test.builds:
99+ if (build.sourcepackagerelease.name == name
100+ and build.processor.name == processor):
101+ return (build, build.buildqueue_record)
102+ return (None, None)
103+
104+
105+def nth_builder(test, build, n):
106+ """Find nth builder that can execute the given build."""
107+ def builder_key(build):
108+ return (build.processor.id,build.is_virtualized)
109+ builder = None
110+ builders = test.builders.get(builder_key(build), [])
111+ try:
112+ builder = builders[n-1]
113+ except IndexError:
114+ pass
115+ return builder
116+
117+
118+def assign_to_builder(test, job_name, builder_number):
119+ """Simulate assigning a build to a builder."""
120+ build, bq = find_job(test, job_name)
121+ builder = nth_builder(test, build, builder_number)
122+ bq.markAsBuilding(builder)
123+
124+
125+def print_build_setup(builds):
126+ """Show the build set-up for a particular test."""
127+ for build in builds:
128+ bq = build.buildqueue_record
129+ spr = build.sourcepackagerelease
130+ print "%12s, p:%5s, v:%5s e:%s *** s:%5s" % (
131+ spr.name, build.processor.name, build.is_virtualized,
132+ bq.estimated_duration, bq.lastscore)
133+
134+
135+class TestBuildQueueBase(TestCaseWithFactory):
136+ """Setup the test publisher and some builders."""
137+
138+ layer = LaunchpadZopelessLayer
139+
140+ def setUp(self):
141+ super(TestBuildQueueBase, self).setUp()
142+ self.publisher = SoyuzTestPublisher()
143+ self.publisher.prepareBreezyAutotest()
144+
145+ # First make nine 'i386' builders.
146+ self.i1 = self.factory.makeBuilder(name='i386-v-1')
147+ self.i2 = self.factory.makeBuilder(name='i386-v-2')
148+ self.i3 = self.factory.makeBuilder(name='i386-v-3')
149+ self.i4 = self.factory.makeBuilder(name='i386-v-4')
150+ self.i5 = self.factory.makeBuilder(name='i386-v-5')
151+ self.i6 = self.factory.makeBuilder(name='i386-n-6', virtualized=False)
152+ self.i7 = self.factory.makeBuilder(name='i386-n-7', virtualized=False)
153+ self.i8 = self.factory.makeBuilder(name='i386-n-8', virtualized=False)
154+ self.i9 = self.factory.makeBuilder(name='i386-n-9', virtualized=False)
155+
156+ # Next make seven 'hppa' builders.
157+ processor_fam = ProcessorFamilySet().getByName('hppa')
158+ proc = processor_fam.processors[0]
159+ self.h1 = self.factory.makeBuilder(name='hppa-v-1', processor=proc)
160+ self.h2 = self.factory.makeBuilder(name='hppa-v-2', processor=proc)
161+ self.h3 = self.factory.makeBuilder(name='hppa-v-3', processor=proc)
162+ self.h4 = self.factory.makeBuilder(name='hppa-v-4', processor=proc)
163+ self.h5 = self.factory.makeBuilder(
164+ name='hppa-n-5', processor=proc, virtualized=False)
165+ self.h6 = self.factory.makeBuilder(
166+ name='hppa-n-6', processor=proc, virtualized=False)
167+ self.h7 = self.factory.makeBuilder(
168+ name='hppa-n-7', processor=proc, virtualized=False)
169+
170+ # Finally make five 'amd64' builders.
171+ processor_fam = ProcessorFamilySet().getByName('amd64')
172+ proc = processor_fam.processors[0]
173+ self.a1 = self.factory.makeBuilder(name='amd64-v-1', processor=proc)
174+ self.a2 = self.factory.makeBuilder(name='amd64-v-2', processor=proc)
175+ self.a3 = self.factory.makeBuilder(name='amd64-v-3', processor=proc)
176+ self.a4 = self.factory.makeBuilder(
177+ name='amd64-n-4', processor=proc, virtualized=False)
178+ self.a5 = self.factory.makeBuilder(
179+ name='amd64-n-5', processor=proc, virtualized=False)
180+
181+ self.builders = dict()
182+ # x86 native
183+ self.builders[(1,False)] = [self.i6, self.i7, self.i8, self.i9]
184+ # x86 virtual
185+ self.builders[(1,True)] = [
186+ self.i1, self.i2, self.i3, self.i4, self.i5]
187+
188+ # amd64 native
189+ self.builders[(2,True)] = [self.a4, self.a5]
190+ # amd64 virtual
191+ self.builders[(2,False)] = [self.a1, self.a2, self.a3]
192+
193+ # hppa native
194+ self.builders[(3,True)] = [self.h5, self.h6, self.h7]
195+ # hppa virtual
196+ self.builders[(3,False)] = [self.h1, self.h2, self.h3, self.h4]
197+
198+ # Ensure all builders are operational.
199+ for builders in self.builders.values():
200+ for builder in builders:
201+ builder.builderok = True
202+ builder.manual = False
203+
204+ # Disable the sample data builders.
205+ getUtility(IBuilderSet)['bob'].builderok = False
206+ getUtility(IBuilderSet)['frog'].builderok = False
207+
208+
209+class TestBuilderData(TestBuildQueueBase):
210+ """Test the retrieval of builder related data. The latter is required
211+ for job dispatch time estimations irrespective of job processor
212+ architecture and virtualization setting."""
213+
214+ def setUp(self):
215+ """Set up some native x86 builds for the test archive."""
216+ super(TestBuilderData, self).setUp()
217+ # The builds will be set up as follows:
218+ #
219+ # gedit, p: 386, v:False e:0:01:00 *** s: 1001
220+ # firefox, p: 386, v:False e:0:02:00 *** s: 1002
221+ # apg, p: 386, v:False e:0:03:00 *** s: 1003
222+ # vim, p: 386, v:False e:0:04:00 *** s: 1004
223+ # gcc, p: 386, v:False e:0:05:00 *** s: 1005
224+ # bison, p: 386, v:False e:0:06:00 *** s: 1006
225+ # flex, p: 386, v:False e:0:07:00 *** s: 1007
226+ # postgres, p: 386, v:False e:0:08:00 *** s: 1008
227+ #
228+ # p=processor, v=virtualized, e=estimated_duration, s=score
229+
230+ # First mark all builds in the sample data as already built.
231+ store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
232+ sample_data = store.find(Build)
233+ for build in sample_data:
234+ build.buildstate = BuildStatus.FULLYBUILT
235+ store.flush()
236+
237+ # We test builds that target a primary archive.
238+ self.non_ppa = self.factory.makeArchive(
239+ name="primary", purpose=ArchivePurpose.PRIMARY)
240+ self.non_ppa.require_virtualized = False
241+
242+ self.builds = []
243+ self.builds.extend(
244+ self.publisher.getPubSource(
245+ sourcename="gedit", status=PackagePublishingStatus.PUBLISHED,
246+ archive=self.non_ppa).createMissingBuilds())
247+ self.builds.extend(
248+ self.publisher.getPubSource(
249+ sourcename="firefox",
250+ status=PackagePublishingStatus.PUBLISHED,
251+ archive=self.non_ppa).createMissingBuilds())
252+ self.builds.extend(
253+ self.publisher.getPubSource(
254+ sourcename="apg", status=PackagePublishingStatus.PUBLISHED,
255+ archive=self.non_ppa).createMissingBuilds())
256+ self.builds.extend(
257+ self.publisher.getPubSource(
258+ sourcename="vim", status=PackagePublishingStatus.PUBLISHED,
259+ archive=self.non_ppa).createMissingBuilds())
260+ self.builds.extend(
261+ self.publisher.getPubSource(
262+ sourcename="gcc", status=PackagePublishingStatus.PUBLISHED,
263+ archive=self.non_ppa).createMissingBuilds())
264+ self.builds.extend(
265+ self.publisher.getPubSource(
266+ sourcename="bison", status=PackagePublishingStatus.PUBLISHED,
267+ archive=self.non_ppa).createMissingBuilds())
268+ self.builds.extend(
269+ self.publisher.getPubSource(
270+ sourcename="flex", status=PackagePublishingStatus.PUBLISHED,
271+ archive=self.non_ppa).createMissingBuilds())
272+ self.builds.extend(
273+ self.publisher.getPubSource(
274+ sourcename="postgres",
275+ status=PackagePublishingStatus.PUBLISHED,
276+ archive=self.non_ppa).createMissingBuilds())
277+ # Set up the builds for test.
278+ score = 1000
279+ duration = 0
280+ for build in self.builds:
281+ score += 1
282+ duration += 60
283+ bq = build.buildqueue_record
284+ bq.lastscore = score
285+ bq.estimated_duration = timedelta(seconds=duration)
286+ # print_build_setup(self.builds)
287+
288+ def test_builder_data(self):
289+ # Make sure the builder numbers are correct. The builder data will
290+ # be the same for all of our builds.
291+ bq = self.builds[0].buildqueue_record
292+ builder_data = bq._getBuilderData()
293+ builders_in_total, builders_for_job, builder_stats = builder_data
294+ self.assertEqual(
295+ builders_in_total, 21, "The total number of builders is wrong.")
296+ self.assertEqual(
297+ builders_for_job, 4,
298+ "[1] The total number of builders that can build the job in "
299+ "question is wrong.")
300+ self.assertEqual(
301+ builder_stats[(1,False)], 4,
302+ "The number of native x86 builders is wrong")
303+ self.assertEqual(
304+ builder_stats[(1,True)], 5,
305+ "The number of virtual x86 builders is wrong")
306+ self.assertEqual(
307+ builder_stats[(2,False)], 2,
308+ "The number of native amd64 builders is wrong")
309+ self.assertEqual(
310+ builder_stats[(2,True)], 3,
311+ "The number of virtual amd64 builders is wrong")
312+ self.assertEqual(
313+ builder_stats[(3,False)], 3,
314+ "The number of native hppa builders is wrong")
315+ self.assertEqual(
316+ builder_stats[(3,True)], 4,
317+ "The number of virtual hppa builders is wrong")
318+ # Disable the native x86 builders.
319+ for builder in self.builders[(1,False)]:
320+ builder.builderok = False
321+ # Get the builder statistics again.
322+ builder_data = bq._getBuilderData()
323+ builders_in_total, builders_for_job, builder_stats = builder_data
324+ # Since all native x86 builders were disabled there are none left
325+ # to build the job.
326+ self.assertEqual(
327+ builders_for_job, 0,
328+ "[2] The total number of builders that can build the job in "
329+ "question is wrong.")
330+ # Re-enable one of them.
331+ for builder in self.builders[(1,False)]:
332+ builder.builderok = True
333+ break
334+ # Get the builder statistics again.
335+ builder_data = bq._getBuilderData()
336+ builders_in_total, builders_for_job, builder_stats = builder_data
337+ # Now there should be one builder available to build the job.
338+ self.assertEqual(
339+ builders_for_job, 1,
340+ "[3] The total number of builders that can build the job in "
341+ "question is wrong.")
342+ # Disable the *virtual* x86 builders -- should not make any
343+ # difference.
344+ for builder in self.builders[(1,True)]:
345+ builder.builderok = False
346+ # Get the builder statistics again.
347+ builder_data = bq._getBuilderData()
348+ builders_in_total, builders_for_job, builder_stats = builder_data
349+ # There should still be one builder available to build the job.
350+ self.assertEqual(
351+ builders_for_job, 1,
352+ "[4] The total number of builders that can build the job in "
353+ "question is wrong.")
354+
355+ def test_free_builder_counts(self):
356+ # Make sure the builder numbers are correct. The builder data will
357+ # be the same for all of our builds.
358+ processor_fam = ProcessorFamilySet().getByName('x86')
359+ proc_386 = processor_fam.processors[0]
360+ build = self.builds[0]
361+ # The build in question is an x86/native one.
362+ self.assertEqual(build.processor.id, proc_386.id)
363+ self.assertEqual(build.is_virtualized, False)
364+ bq = build.buildqueue_record
365+ builder_data = bq._getBuilderData()
366+ builders_in_total, builders_for_job, builder_stats = builder_data
367+ # We have 4 x86 native builders.
368+ self.assertEqual(
369+ builder_stats[(proc_386.id,False)], 4,
370+ "The number of native x86 builders is wrong")
371+ # Initially all 4 builders are free.
372+ free_count = bq._freeBuildersCount(
373+ build.processor, build.is_virtualized)
374+ self.assertEqual(free_count, 4)
375+ # Once we assign a build to one of them we should see the free
376+ # builders count drop by one.
377+ assign_to_builder(self, 'postgres', 1)
378+ free_count = bq._freeBuildersCount(
379+ build.processor, build.is_virtualized)
380+ self.assertEqual(free_count, 3)
381+ # When we assign another build to one of them we should see the free
382+ # builders count drop by one again.
383+ assign_to_builder(self, 'gcc', 2)
384+ free_count = bq._freeBuildersCount(
385+ build.processor, build.is_virtualized)
386+ self.assertEqual(free_count, 2)
387+ # Let's use up another builder.
388+ assign_to_builder(self, 'apg', 3)
389+ free_count = bq._freeBuildersCount(
390+ build.processor, build.is_virtualized)
391+ self.assertEqual(free_count, 1)
392+ # And now for the last one.
393+ assign_to_builder(self, 'flex', 4)
394+ free_count = bq._freeBuildersCount(
395+ build.processor, build.is_virtualized)
396+ self.assertEqual(free_count, 0)
397+ # If we reset the 'flex' build the builder that was assigned to it
398+ # will be free again.
399+ build, bq = find_job(self, 'flex')
400+ bq.reset()
401+ free_count = bq._freeBuildersCount(
402+ build.processor, build.is_virtualized)
403+ self.assertEqual(free_count, 1)