Launchpad itself

Merge lp://staging/~al-maisan/launchpad/builder-data-502841 into lp://staging/launchpad

  1. builder-data-502841
  2. Merge into devel
Proposed by Muharem Hrnjadovic
Status: Merged
Merged at revision: not available
Proposed branch: lp://staging/~al-maisan/launchpad/builder-data-502841
Merge into: lp://staging/launchpad
Diff against target: 403 lines (+388/-0)
2 files modified
lib/lp/soyuz/model/buildqueue.py (+56/-0)
lib/lp/soyuz/tests/test_buildqueue.py (+332/-0)
To merge this branch: bzr merge lp://staging/~al-maisan/launchpad/builder-data-502841
Reviewer Review Type Date Requested Status
Eleanor Berger (community) Approve
Review via email: mp+16774@code.staging.launchpad.net
To post a comment you must log in.
Revision history for this message
Muharem Hrnjadovic (al-maisan) wrote :

Hello there!

This is another piece required to get us to the point where we can estimate
job dispatch times in the Soyuz build farm irrespective of job type.

The code added

  - retrieves data about the builders that are available in the system for
    the various processor architectures and virtualization settings.
  - check how many builders are free/available to take on a job for a given
    build (processor and virtualization setting).

Tests to run:

    bin/test -vv -t TestBuilderData

No "make lint" errors or warnings.

Revision history for this message
Eleanor Berger (intellectronica) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'lib/lp/soyuz/model/buildqueue.py'
--- lib/lp/soyuz/model/buildqueue.py 2009-12-10 20:06:11 +0000
+++ lib/lp/soyuz/model/buildqueue.py 2010-01-04 14:33:16 +0000
@@ -156,6 +156,62 @@
156 """See `IBuildQueue`."""156 """See `IBuildQueue`."""
157 self.job.date_started = timestamp157 self.job.date_started = timestamp
158158
159 def _getBuilderData(self):
160 """How many working builders are there, how are they configured?"""
161 # Please note: this method will send only one request to the database.
162
163 store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
164 my_processor = self.specific_job.processor
165 my_virtualized = self.specific_job.virtualized
166
167 # We need to know the total number of builders as well as the
168 # number of builders that can run the job of interest (JOI).
169 # If the JOI is processor independent these builder counts will
170 # have the same value.
171 builder_data = """
172 SELECT processor, virtualized, COUNT(id) FROM builder
173 WHERE builderok = TRUE AND manual = FALSE
174 GROUP BY processor, virtualized;
175 """
176 results = store.execute(builder_data).get_all()
177
178 builder_stats = dict()
179 builders_in_total = builders_for_job = 0
180 for processor, virtualized, count in results:
181 if my_processor is not None:
182 if (my_processor.id == processor and
183 my_virtualized == virtualized):
184 # The job on hand can only run on builders with a
185 # particular processor/virtualization combination and
186 # this is how many of these we have.
187 builders_for_job = count
188 builders_in_total += count
189 builder_stats[(processor, virtualized)] = count
190 if my_processor is None:
191 # The job of interest (JOI) is processor independent.
192 builders_for_job = builders_in_total
193
194 return (builders_in_total, builders_for_job, builder_stats)
195
196 def _freeBuildersCount(self, processor, virtualized):
197 """How many builders capable of running jobs for the given processor
198 and virtualization combination are idle/free at present?"""
199 query = """
200 SELECT COUNT(id) FROM builder
201 WHERE
202 builderok = TRUE AND manual = FALSE
203 AND id NOT IN (
204 SELECT builder FROM BuildQueue WHERE builder IS NOT NULL)
205 """
206 if processor is not None:
207 query += """
208 AND processor = %s AND virtualized = %s
209 """ % sqlvalues(processor, virtualized)
210 store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
211 result_set = store.execute(query)
212 free_builders = result_set.get_one()[0]
213 return free_builders
214
159215
160class BuildQueueSet(object):216class BuildQueueSet(object):
161 """Utility to deal with BuildQueue content class."""217 """Utility to deal with BuildQueue content class."""
162218
=== added file 'lib/lp/soyuz/tests/test_buildqueue.py'
--- lib/lp/soyuz/tests/test_buildqueue.py 1970-01-01 00:00:00 +0000
+++ lib/lp/soyuz/tests/test_buildqueue.py 2010-01-04 14:33:16 +0000
@@ -0,0 +1,332 @@
1# Copyright 2009 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).
3# pylint: disable-msg=C0324
4
5"""Test BuildQueue features."""
6
7from datetime import timedelta
8
9from zope.component import getUtility
10
11from canonical.launchpad.webapp.interfaces import (
12 IStoreSelector, MAIN_STORE, DEFAULT_FLAVOR)
13from canonical.testing import LaunchpadZopelessLayer
14
15from lp.soyuz.interfaces.archive import ArchivePurpose
16from lp.soyuz.interfaces.build import BuildStatus
17from lp.soyuz.interfaces.builder import IBuilderSet
18from lp.soyuz.model.processor import ProcessorFamilySet
19from lp.soyuz.interfaces.publishing import PackagePublishingStatus
20from lp.soyuz.model.build import Build
21from lp.soyuz.tests.test_publishing import SoyuzTestPublisher
22from lp.testing import TestCaseWithFactory
23
24
25def find_job(test, name, processor='386'):
26 """Find build and queue instance for the given source and processor."""
27 for build in test.builds:
28 if (build.sourcepackagerelease.name == name
29 and build.processor.name == processor):
30 return (build, build.buildqueue_record)
31 return (None, None)
32
33
34def nth_builder(test, build, n):
35 """Find nth builder that can execute the given build."""
36 def builder_key(build):
37 return (build.processor.id,build.is_virtualized)
38 builder = None
39 builders = test.builders.get(builder_key(build), [])
40 try:
41 builder = builders[n-1]
42 except IndexError:
43 pass
44 return builder
45
46
47def assign_to_builder(test, job_name, builder_number):
48 """Simulate assigning a build to a builder."""
49 build, bq = find_job(test, job_name)
50 builder = nth_builder(test, build, builder_number)
51 bq.markAsBuilding(builder)
52
53
54def print_build_setup(builds):
55 """Show the build set-up for a particular test."""
56 for build in builds:
57 bq = build.buildqueue_record
58 spr = build.sourcepackagerelease
59 print "%12s, p:%5s, v:%5s e:%s *** s:%5s" % (
60 spr.name, build.processor.name, build.is_virtualized,
61 bq.estimated_duration, bq.lastscore)
62
63
64class TestBuildQueueBase(TestCaseWithFactory):
65 """Setup the test publisher and some builders."""
66
67 layer = LaunchpadZopelessLayer
68
69 def setUp(self):
70 super(TestBuildQueueBase, self).setUp()
71 self.publisher = SoyuzTestPublisher()
72 self.publisher.prepareBreezyAutotest()
73
74 # First make nine 'i386' builders.
75 self.i1 = self.factory.makeBuilder(name='i386-v-1')
76 self.i2 = self.factory.makeBuilder(name='i386-v-2')
77 self.i3 = self.factory.makeBuilder(name='i386-v-3')
78 self.i4 = self.factory.makeBuilder(name='i386-v-4')
79 self.i5 = self.factory.makeBuilder(name='i386-v-5')
80 self.i6 = self.factory.makeBuilder(name='i386-n-6', virtualized=False)
81 self.i7 = self.factory.makeBuilder(name='i386-n-7', virtualized=False)
82 self.i8 = self.factory.makeBuilder(name='i386-n-8', virtualized=False)
83 self.i9 = self.factory.makeBuilder(name='i386-n-9', virtualized=False)
84
85 # Next make seven 'hppa' builders.
86 processor_fam = ProcessorFamilySet().getByName('hppa')
87 proc = processor_fam.processors[0]
88 self.h1 = self.factory.makeBuilder(name='hppa-v-1', processor=proc)
89 self.h2 = self.factory.makeBuilder(name='hppa-v-2', processor=proc)
90 self.h3 = self.factory.makeBuilder(name='hppa-v-3', processor=proc)
91 self.h4 = self.factory.makeBuilder(name='hppa-v-4', processor=proc)
92 self.h5 = self.factory.makeBuilder(
93 name='hppa-n-5', processor=proc, virtualized=False)
94 self.h6 = self.factory.makeBuilder(
95 name='hppa-n-6', processor=proc, virtualized=False)
96 self.h7 = self.factory.makeBuilder(
97 name='hppa-n-7', processor=proc, virtualized=False)
98
99 # Finally make five 'amd64' builders.
100 processor_fam = ProcessorFamilySet().getByName('amd64')
101 proc = processor_fam.processors[0]
102 self.a1 = self.factory.makeBuilder(name='amd64-v-1', processor=proc)
103 self.a2 = self.factory.makeBuilder(name='amd64-v-2', processor=proc)
104 self.a3 = self.factory.makeBuilder(name='amd64-v-3', processor=proc)
105 self.a4 = self.factory.makeBuilder(
106 name='amd64-n-4', processor=proc, virtualized=False)
107 self.a5 = self.factory.makeBuilder(
108 name='amd64-n-5', processor=proc, virtualized=False)
109
110 self.builders = dict()
111 # x86 native
112 self.builders[(1,False)] = [self.i6, self.i7, self.i8, self.i9]
113 # x86 virtual
114 self.builders[(1,True)] = [
115 self.i1, self.i2, self.i3, self.i4, self.i5]
116
117 # amd64 native
118 self.builders[(2,True)] = [self.a4, self.a5]
119 # amd64 virtual
120 self.builders[(2,False)] = [self.a1, self.a2, self.a3]
121
122 # hppa native
123 self.builders[(3,True)] = [self.h5, self.h6, self.h7]
124 # hppa virtual
125 self.builders[(3,False)] = [self.h1, self.h2, self.h3, self.h4]
126
127 # Ensure all builders are operational.
128 for builders in self.builders.values():
129 for builder in builders:
130 builder.builderok = True
131 builder.manual = False
132
133 # Disable the sample data builders.
134 getUtility(IBuilderSet)['bob'].builderok = False
135 getUtility(IBuilderSet)['frog'].builderok = False
136
137
138class TestBuilderData(TestBuildQueueBase):
139 """Test the retrieval of builder related data. The latter is required
140 for job dispatch time estimations irrespective of job processor
141 architecture and virtualization setting."""
142
143 def setUp(self):
144 """Set up some native x86 builds for the test archive."""
145 super(TestBuilderData, self).setUp()
146 # The builds will be set up as follows:
147 #
148 # gedit, p: 386, v:False e:0:01:00 *** s: 1001
149 # firefox, p: 386, v:False e:0:02:00 *** s: 1002
150 # apg, p: 386, v:False e:0:03:00 *** s: 1003
151 # vim, p: 386, v:False e:0:04:00 *** s: 1004
152 # gcc, p: 386, v:False e:0:05:00 *** s: 1005
153 # bison, p: 386, v:False e:0:06:00 *** s: 1006
154 # flex, p: 386, v:False e:0:07:00 *** s: 1007
155 # postgres, p: 386, v:False e:0:08:00 *** s: 1008
156 #
157 # p=processor, v=virtualized, e=estimated_duration, s=score
158
159 # First mark all builds in the sample data as already built.
160 store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
161 sample_data = store.find(Build)
162 for build in sample_data:
163 build.buildstate = BuildStatus.FULLYBUILT
164 store.flush()
165
166 # We test builds that target a primary archive.
167 self.non_ppa = self.factory.makeArchive(
168 name="primary", purpose=ArchivePurpose.PRIMARY)
169 self.non_ppa.require_virtualized = False
170
171 self.builds = []
172 self.builds.extend(
173 self.publisher.getPubSource(
174 sourcename="gedit", status=PackagePublishingStatus.PUBLISHED,
175 archive=self.non_ppa).createMissingBuilds())
176 self.builds.extend(
177 self.publisher.getPubSource(
178 sourcename="firefox",
179 status=PackagePublishingStatus.PUBLISHED,
180 archive=self.non_ppa).createMissingBuilds())
181 self.builds.extend(
182 self.publisher.getPubSource(
183 sourcename="apg", status=PackagePublishingStatus.PUBLISHED,
184 archive=self.non_ppa).createMissingBuilds())
185 self.builds.extend(
186 self.publisher.getPubSource(
187 sourcename="vim", status=PackagePublishingStatus.PUBLISHED,
188 archive=self.non_ppa).createMissingBuilds())
189 self.builds.extend(
190 self.publisher.getPubSource(
191 sourcename="gcc", status=PackagePublishingStatus.PUBLISHED,
192 archive=self.non_ppa).createMissingBuilds())
193 self.builds.extend(
194 self.publisher.getPubSource(
195 sourcename="bison", status=PackagePublishingStatus.PUBLISHED,
196 archive=self.non_ppa).createMissingBuilds())
197 self.builds.extend(
198 self.publisher.getPubSource(
199 sourcename="flex", status=PackagePublishingStatus.PUBLISHED,
200 archive=self.non_ppa).createMissingBuilds())
201 self.builds.extend(
202 self.publisher.getPubSource(
203 sourcename="postgres",
204 status=PackagePublishingStatus.PUBLISHED,
205 archive=self.non_ppa).createMissingBuilds())
206 # Set up the builds for test.
207 score = 1000
208 duration = 0
209 for build in self.builds:
210 score += 1
211 duration += 60
212 bq = build.buildqueue_record
213 bq.lastscore = score
214 bq.estimated_duration = timedelta(seconds=duration)
215 # print_build_setup(self.builds)
216
217 def test_builder_data(self):
218 # Make sure the builder numbers are correct. The builder data will
219 # be the same for all of our builds.
220 bq = self.builds[0].buildqueue_record
221 builder_data = bq._getBuilderData()
222 builders_in_total, builders_for_job, builder_stats = builder_data
223 self.assertEqual(
224 builders_in_total, 21, "The total number of builders is wrong.")
225 self.assertEqual(
226 builders_for_job, 4,
227 "[1] The total number of builders that can build the job in "
228 "question is wrong.")
229 self.assertEqual(
230 builder_stats[(1,False)], 4,
231 "The number of native x86 builders is wrong")
232 self.assertEqual(
233 builder_stats[(1,True)], 5,
234 "The number of virtual x86 builders is wrong")
235 self.assertEqual(
236 builder_stats[(2,False)], 2,
237 "The number of native amd64 builders is wrong")
238 self.assertEqual(
239 builder_stats[(2,True)], 3,
240 "The number of virtual amd64 builders is wrong")
241 self.assertEqual(
242 builder_stats[(3,False)], 3,
243 "The number of native hppa builders is wrong")
244 self.assertEqual(
245 builder_stats[(3,True)], 4,
246 "The number of virtual hppa builders is wrong")
247 # Disable the native x86 builders.
248 for builder in self.builders[(1,False)]:
249 builder.builderok = False
250 # Get the builder statistics again.
251 builder_data = bq._getBuilderData()
252 builders_in_total, builders_for_job, builder_stats = builder_data
253 # Since all native x86 builders were disabled there are none left
254 # to build the job.
255 self.assertEqual(
256 builders_for_job, 0,
257 "[2] The total number of builders that can build the job in "
258 "question is wrong.")
259 # Re-enable one of them.
260 for builder in self.builders[(1,False)]:
261 builder.builderok = True
262 break
263 # Get the builder statistics again.
264 builder_data = bq._getBuilderData()
265 builders_in_total, builders_for_job, builder_stats = builder_data
266 # Now there should be one builder available to build the job.
267 self.assertEqual(
268 builders_for_job, 1,
269 "[3] The total number of builders that can build the job in "
270 "question is wrong.")
271 # Disable the *virtual* x86 builders -- should not make any
272 # difference.
273 for builder in self.builders[(1,True)]:
274 builder.builderok = False
275 # Get the builder statistics again.
276 builder_data = bq._getBuilderData()
277 builders_in_total, builders_for_job, builder_stats = builder_data
278 # There should still be one builder available to build the job.
279 self.assertEqual(
280 builders_for_job, 1,
281 "[4] The total number of builders that can build the job in "
282 "question is wrong.")
283
284 def test_free_builder_counts(self):
285 # Make sure the builder numbers are correct. The builder data will
286 # be the same for all of our builds.
287 processor_fam = ProcessorFamilySet().getByName('x86')
288 proc_386 = processor_fam.processors[0]
289 build = self.builds[0]
290 # The build in question is an x86/native one.
291 self.assertEqual(build.processor.id, proc_386.id)
292 self.assertEqual(build.is_virtualized, False)
293 bq = build.buildqueue_record
294 builder_data = bq._getBuilderData()
295 builders_in_total, builders_for_job, builder_stats = builder_data
296 # We have 4 x86 native builders.
297 self.assertEqual(
298 builder_stats[(proc_386.id,False)], 4,
299 "The number of native x86 builders is wrong")
300 # Initially all 4 builders are free.
301 free_count = bq._freeBuildersCount(
302 build.processor, build.is_virtualized)
303 self.assertEqual(free_count, 4)
304 # Once we assign a build to one of them we should see the free
305 # builders count drop by one.
306 assign_to_builder(self, 'postgres', 1)
307 free_count = bq._freeBuildersCount(
308 build.processor, build.is_virtualized)
309 self.assertEqual(free_count, 3)
310 # When we assign another build to one of them we should see the free
311 # builders count drop by one again.
312 assign_to_builder(self, 'gcc', 2)
313 free_count = bq._freeBuildersCount(
314 build.processor, build.is_virtualized)
315 self.assertEqual(free_count, 2)
316 # Let's use up another builder.
317 assign_to_builder(self, 'apg', 3)
318 free_count = bq._freeBuildersCount(
319 build.processor, build.is_virtualized)
320 self.assertEqual(free_count, 1)
321 # And now for the last one.
322 assign_to_builder(self, 'flex', 4)
323 free_count = bq._freeBuildersCount(
324 build.processor, build.is_virtualized)
325 self.assertEqual(free_count, 0)
326 # If we reset the 'flex' build the builder that was assigned to it
327 # will be free again.
328 build, bq = find_job(self, 'flex')
329 bq.reset()
330 free_count = bq._freeBuildersCount(
331 build.processor, build.is_virtualized)
332 self.assertEqual(free_count, 1)