Name | Total Lines | Lines of Code | Total Coverage | Code Coverage |
---|---|---|---|---|
kci/builder.rb | 451 | 222 | 78.71%
|
56.76%
|
Code reported as executed by Ruby looks like this...and this: this line is also marked as covered.Lines considered as run by rcov, but not reported by Ruby, look like this,and this: these lines were inferred by rcov (using simple heuristics).Finally, here's a line marked as not executed.
2 #!/usr/bin/env ruby |
3 |
4 require 'date' |
5 require 'fileutils' |
6 require 'json' |
7 require 'timeout' |
8 |
9 require_relative '../lib/ci/build_source' |
10 require_relative '../lib/ci/build_version' |
11 require_relative '../lib/debian/changelog' |
12 require_relative '../lib/debian/dsc_arch_twiddle' |
13 require_relative '../lib/cmake_parser' |
14 require_relative '../lib/kci' |
15 require_relative '../lib/lint/control' |
16 require_relative '../lib/lint/result' |
17 require_relative '../lib/lint/series' |
18 require_relative '../lib/lint/symbols' |
19 |
20 =begin |
21 -> build_source |
22 - copy upstream source |
23 - generate tarball |
24 - copy packaging source |
25 - fiddle |
26 - update changelog |
27 - dpkg-buildpackage -S |
28 -> sign_source [elevated - possibly part of build_source] |
29 - debsign |
30 -> build binary |
31 - upload_source |
32 - wait_for_launchpad |
33 - download_logs |
34 -> check_logs |
35 - cmake |
36 - lintian |
37 - symbols |
38 - qml-checker |
39 -> update_symbols |
40 =end |
41 |
42 class KCIBuilder |
43 class CoverageError < Exception; end |
44 |
45 DPUTCONF = '/var/lib/jenkins/tooling/dput.cf' |
46 KEYID = '6A0C5BF2' |
47 Project = Struct.new(:series, :stability, :name) |
48 |
49 def self.testing=(testing) |
50 @testing = testing |
51 end |
52 |
53 def self.testing |
54 @testing ||= false |
55 end |
56 |
57 def self.build_and_publish(project, source) |
58 Dir.chdir('build') do |
59 # Upload likes to get stuck, so we do timeout control to prevent all of |
60 # builder from getting stuck. |
61 # We try to dput two times in a row giving it first 30 minutes and then |
62 # 15 minutes to complete. If it didn't manage to upload after that we |
63 # ignore the package and move on. |
64 `cp -rf /var/lib/jenkins/.ssh /root/` |
65 `chown -Rv root:root /root/.ssh` |
66 if `ssh-keygen -F ppa.launchpad.net`.strip.empty? |
67 `ssh-keyscan -H ppa.launchpad.net >> ~/.ssh/known_hosts` |
68 end |
69 dput = "dput -d -c #{DPUTCONF} #{@ppa} #{source.name}_#{source.build_version.tar}*.changes" |
70 success = false |
71 4.times do |count| |
72 # Note: count starts at 0 ;) |
73 if timeout_spawn(dput, 60 * (30.0 / (count + 1))) |
74 puts "spawn timeout goody" |
75 success = true |
76 break |
77 end |
78 puts "spawn timout badly" |
79 sleep(60) # Sleep for a minute |
80 end |
81 abort '\t\t !!!!!!!!!!!! dput failed two times !!!!!!!!!!!!' unless success |
82 Dir.chdir('..') do # main dir |
83 puts "before require" |
84 require_relative 'source_publisher' |
85 puts "after require; before new" |
86 publisher = SourcePublisher.new(source.name, source.version, project.stability) |
87 puts "after new" |
88 abort 'PPA Build Failed' unless publisher.wait |
89 # Write upload data to file, we perhaps want to do something outside the |
90 # build container. |
91 data = { name: source.name, |
92 version: source.version, |
93 type: project.stability } |
94 File.write('source.json', JSON.generate(data)) |
95 end |
96 end |
97 end |
98 |
99 def self.run |
100 ENV['GNUPGHOME'] = '/var/lib/jenkins/tooling/gnupg' |
101 |
102 $stdout = $stderr |
103 |
104 # get basename, distro series, unstable/stable |
105 components = ARGV.fetch(0).split('_') |
106 unless components.size == 3 |
107 abort 'Did not get a valid project identifier via ARGV0' |
108 end |
109 project = Project.new(components.fetch(0), |
110 components.fetch(1), |
111 components.fetch(2)) |
112 |
113 @ppa = "ppa:kubuntu-ci/#{project.stability}" |
114 |
115 # PWD |
116 abort 'Could not change dir to ARGV1' unless Dir.chdir(ARGV.fetch(1)) |
117 @workspace_path = ARGV.fetch(1) |
118 |
119 # Workaround for docker not having suidmaps. We run as root in the docker |
120 # which will result in uid/gid of written things to be 0 rather than whatever |
121 # jenkins has. So instead we have a fake jenkins user in the docker we can |
122 # chmod to. This ultimately ensures that the owernship is using the uid of |
123 # the host jenkins (equal to docker jenkins) such that we don't end up with |
124 # stuff owned by others. |
125 at_exit do |
126 FileUtils.chown_R('jenkins', 'jenkins', @workspace_path, verbose: true) |
127 end unless testing |
128 |
129 source = VcsSourceBuilder.new(release: project.series).run |
130 |
131 # Mangle dsc to not do ARM builds unless explicitly enabled. |
132 # With hundreds of distinct sources on CI, building all of them on three |
133 # architectures, of which one is not too commonly used, is extremly excessive. |
134 # Instead, by default we only build on the common architectures with extra |
135 # architectures needing to be enabled explicitly. |
136 # To achieve this mangle the Architecture field in the control file. |
137 # If it contains an uncommon arch -> remove it -> if it is empty now, abort |
138 # If it contains any -> replace with !uncommon |
139 # This is a cheapster hack implementation to avoid having to implement write |
140 # support in Debian control. |
141 begin |
142 Debian::DSCArch.twiddle!('build/') |
143 rescue Debian::DSCArch::Error => e |
144 # NOTE: this can raise a number of errors and we want them all to be fatal |
145 # to prevent unhandled dscs or completely empty architectures from |
146 # getting uploaded. |
147 abort e |
148 end |
149 |
150 fail CoverageError, 'Testing disabled after arch twiddle' if testing |
151 |
152 # Sign |
153 Dir.chdir('build/') do |
154 changes = Dir.glob('*.changes') |
155 abort "Expected only one changes file #{changes}" if changes.size != 1 |
156 unless system("debsign -k#{KEYID} #{changes[0]}") |
157 abort 'Failed to sign the source.' |
158 end |
159 end |
160 |
161 build_and_publish(project, source) |
162 |
163 unless File.exist?('logs/i386.log') |
164 puts "found no logs" |
165 exit 0 |
166 end |
167 |
168 # We need discrete arrays of both logs and architectures they represent |
169 # to make sure we process them in the correct order when updating symbols. |
170 logs = [] |
171 architectures_with_log = [] |
172 Dir.chdir('logs/') do |
173 # `gunzip *.log.gz` |
174 Dir.glob('*.log').each do |log| |
175 logs << "#{@workspace_path}/logs/#{log}" |
176 architectures_with_log << File.basename(log, '.log') |
177 end |
178 end |
179 |
180 archindep = File.read('archindep').strip rescue 'amd64' # Get archindep from PPA script. |
181 log_data = File.open("logs/#{archindep}.log").read |
182 |
183 updated_symbols = update_symbols(log_data, logs, architectures_with_log, |
184 project, source) |
185 |
186 puts_cmake(log_data, source.name) |
187 puts_list_missing(log_data) |
188 puts_lintian(log_data, updated_symbols: updated_symbols) |
189 |
190 # Lint control file |
191 results = [] |
192 results << Lint::Control.new('packaging').lint |
193 results << Lint::Series.new('packaging').lint |
194 results << Lint::Symbols.new('packaging').lint |
195 |
196 Lint::ResultLogger.new(results).log |
197 |
198 # TODO: this script currently does not impact the build results nor does it |
199 # create parsable output |
200 qmlsrcs = %w( |
201 bluez-qt |
202 breeeze |
203 kactivities |
204 kalgebra |
205 kanagram |
206 kate |
207 kbreakout |
208 kdeplasma-addons |
209 kdeclarative |
210 kinfocenter |
211 koko |
212 kscreen |
213 ktp-desktop-applets |
214 kwin |
215 libkdegames |
216 plasma-framework |
217 plasma-desktop |
218 plasma-mediacenter |
219 plasma-nm |
220 plasma-sdk |
221 plasma-volume-control |
222 print-manager |
223 purpose |
224 milou |
225 muon |
226 ) |
227 if !Dir.glob('source/**/*.qml').empty? && qmlsrcs.include?(project.name) |
228 require_relative 'lib/qml_dependency_verifier' |
229 |
230 dep_verify = QMLDependencyVerifier.new |
231 dep_verify.add_ppa |
232 missing_modules = dep_verify.missing_modules |
233 missing_modules.each do |package, modules| |
234 puts_warning "#{package} has missing dependencies..." |
235 modules.uniq! { |mod| { mod.identifier => mod.version } } |
236 modules.each do |mod| |
237 puts_info " #{mod} not found." |
238 puts_info ' looked for:' |
239 mod.import_paths.each do |path| |
240 puts_info " - #{path}" |
241 end |
242 end |
243 end |
244 end |
245 end |
246 |
247 # Upload |
248 def self.timeout_spawn(cmd, timeout) |
249 thread = Thread.new do |
250 loop do |
251 warn '---- ps aux ----' |
252 warn `ps aux |grep dput` |
253 warn '----' |
254 sleep 60 |
255 end |
256 end |
257 |
258 pid = Process.spawn(cmd, pgroup: true) |
259 begin |
260 puts "waitpid with timeout" |
261 Timeout.timeout(timeout) do |
262 Process.waitpid(pid, 0) |
263 puts "return wait pid" |
264 return ($?.exitstatus == 0) |
265 end |
266 rescue Timeout::Error |
267 puts "timeout error" |
268 Process.kill(15, -Process.getpgid(pid)) |
269 return false |
270 end |
271 ensure |
272 thread.kill |
273 end |
274 |
275 def self.segmentify(data, start_marker, end_marker) |
276 warn 'Compatibility function segmentify called. BuildLogSegmenter should be' \ |
277 ' used instead.' |
278 BuildLogSegmenter.segmentify(data, start_marker, end_marker) |
279 end |
280 |
281 def self.puts_kci(type, str) |
282 Lint::ResultLogger.puts_kci(type, str) |
283 end |
284 |
285 def self.puts_error(str) |
286 puts_kci('E', str) |
287 end |
288 |
289 def self.puts_warning(str) |
290 puts_kci('W', str) |
291 end |
292 |
293 def self.puts_info(str) |
294 puts_kci('I', str) |
295 end |
296 |
297 def self.ignore?(array_of_ignores, thing_to_check) |
298 array_of_ignores.each do |possibly_ignore| |
299 if thing_to_check == possibly_ignore || |
300 thing_to_check.start_with?(possibly_ignore) || |
301 thing_to_check.start_with?(possibly_ignore.chomp('*')) |
302 return true |
303 end |
304 end |
305 false |
306 end |
307 |
308 def self.puts_cmake(data, source_name) |
309 # Bit of compat code. Very lovely... NOT |
310 parser = CMakeParser.new(data) |
311 unless parser.valid |
312 puts_info 'CMakeParser could not parse the log correctly. Aborting parser.' |
313 return |
314 end |
315 missing = parser.missing |
316 warnings = parser.warnings |
317 disabled_features = parser.disabled_features |
318 |
319 puts 'KCI::CMAKE' |
320 |
321 # Force an ignore on Qt5TextToSpeech, it's not released as of right now. |
322 # 2015-03-05 |
323 ignore_missing = %w(Qt5TextToSpeech) |
324 cmake_ignore_path = "#{@workspace_path}/packaging/debian/meta/cmake-ignore" |
325 if File.exist?(cmake_ignore_path) |
326 ignore_missing += File.read(cmake_ignore_path).strip.split("\n") |
327 end |
328 |
329 missing.each do |dep| |
330 next if ignore?(ignore_missing, dep) |
331 puts_warning("Missing Dep: #{dep}") |
332 end |
333 |
334 warnings.each do |warning| |
335 puts_warning(warning) |
336 end |
337 |
338 disabled_features.each do |disabled_feature| |
339 next if ignore?(ignore_missing, disabled_feature) |
340 puts_warning("Disabled Feature: #{disabled_feature}") |
341 end |
342 end |
343 |
344 def self.puts_list_missing(data) |
345 data = segmentify(data, |
346 "=== Start list-missing\n", |
347 "=== End list-missing\n") |
348 return if data.empty? |
349 puts 'KCI::MISSING' |
350 data.each do |line| |
351 # Missing files are always considered errors |
352 puts_error(line) |
353 end |
354 end |
355 |
356 def self.puts_lintian(data, updated_symbols: false) |
357 data = segmentify(data, "=== Start lintian\n", "=== End lintian\n") |
358 return if data.empty? |
359 puts 'KCI::LINTIAN' |
360 data.each do |line| |
361 next if line.start_with?('warning: ') # random warnings from lintian itself |
362 |
363 # Package names can easily go beyond what shit can suck on, so gag it. |
364 next if line.include?('source-package-component-has-long-file-name') |
365 next if line.include?('package-has-long-file-name') |
366 |
367 # We really do not care about standards versions for now. They only ever get |
368 # bumped by the pkg-kde team anyway. |
369 next if line.include?('out-of-date-standards-version') |
370 next if line.include?('newer-standards-version') |
371 |
372 # We package an enormous amount of GUI apps without manpages (in fact |
373 # they arguably wouldn't even make sense what with being GUI apps). So |
374 # ignore any and all manpage warnings to save Harald from having to override |
375 # them in every single application repository. |
376 next if line.include?('binary-without-manpage') |
377 |
378 # Fuck you |
379 next if line.include?('debian-revision-should-not-be-zero') |
380 next if line.include?('bad-distribution-in-changes-file wily') |
381 next if line.include?('not-binnmuable-any-depends-all') |
382 |
383 # Do not print symbols warnings if we already auto-updated. |
384 if updated_symbols && |
385 line.include?('symbols-file-contains-current-version-with-debian-revision') |
386 next |
387 end |
388 |
389 case line[0..1] |
390 when 'W:' |
391 puts_warning(line) |
392 next |
393 when 'E:' |
394 puts_error(line) |
395 next |
396 end |
397 puts_info(line) |
398 end |
399 end |
400 |
401 def self.update_symbols(log_data, logs, architectures_with_log, project, |
402 source) |
403 updated_symbols = false |
404 # FIXME: stability wtf |
405 gensymbols_regex = %r{dpkg-gensymbols: warning: (.*)/symbols doesn't match completely debian/(.*).symbols} |
406 if project.series == KCI.latest_series && log_data.match(gensymbols_regex) |
407 puts 'KCI::SYMBOLS' |
408 if log_data.include?('warning: some symbols or patterns disappeared in the symbols file') |
409 puts_error('It would very much appear that symbols have been retracted') |
410 else |
411 match = log_data.match(/--- debian\/(.*).symbols/) |
412 if match && match.size > 1 |
413 Dir.chdir('packaging') do |
414 system('git config --global user.email "kubuntu-ci@lists.launchpad.net"') |
415 system('git config --global user.name "Kubuntu CI"') |
416 system('git config core.sparsecheckout') |
417 system("git checkout -f remotes/packaging/kubuntu_#{project.stability}") |
418 system('git branch -a') |
419 system('git status') |
420 system('git reset --hard') |
421 captures = match.captures |
422 captures.each do |lib_package| |
423 puts "pkgkde-symbolshelper batchpatch -v #{source.build_version.base} -c #{architectures_with_log.join(',')} #{logs.join(' ')}" |
424 system("pkgkde-symbolshelper batchpatch -v #{source.build_version.base} -c #{architectures_with_log.join(',')} #{logs.join(' ')}") |
425 updated_symbols = ($? == 0) |
426 puts_info("Auto-updated symbols of #{lib_package}") |
427 end |
428 # Username et al apparently is somehow coming from .git or something |
429 # apparently |
430 system('git status') |
431 system('git --no-pager diff') |
432 system('git commit -a -m "Automatic symbol update"') |
433 end |
434 else |
435 puts_error('Failed to update symbols as the package name(s) could not be'\ |
436 ' parsed.') |
437 end |
438 end |
439 end |
440 updated_symbols |
441 end |
442 end |
443 |
444 if __FILE__ == $PROGRAM_NAME |
445 File.open('/etc/apt/apt.conf.d/apt-cacher', 'w') do |file| |
446 file.puts('Acquire::http { Proxy "http://10.0.3.1:3142"; };') |
447 end |
448 |
449 `apt-get install -y xz-utils dpkg-dev ruby dput debhelper pkg-kde-tools devscripts python-launchpadlib ubuntu-dev-tools git` |
450 |
451 KCIBuilder.run |
452 end |
Generated on 2015-10-08 11:49:59 +0200 with SimpleCov-RCov 0.2.3