Ci-tooling C0 Coverage Information - Simploco - RCov

kci/builder.rb

Name Total Lines Lines of Code Total Coverage Code Coverage
kci/builder.rb 451 222
78.71%
56.76%

Key

Code reported as executed by Ruby looks like this...and this: this line is also marked as covered.Lines considered as run by rcov, but not reported by Ruby, look like this,and this: these lines were inferred by rcov (using simple heuristics).Finally, here's a line marked as not executed.

Coverage Details

2 #!/usr/bin/env ruby
3 
4 require 'date'
5 require 'fileutils'
6 require 'json'
7 require 'timeout'
8 
9 require_relative '../lib/ci/build_source'
10 require_relative '../lib/ci/build_version'
11 require_relative '../lib/debian/changelog'
12 require_relative '../lib/debian/dsc_arch_twiddle'
13 require_relative '../lib/cmake_parser'
14 require_relative '../lib/kci'
15 require_relative '../lib/lint/control'
16 require_relative '../lib/lint/result'
17 require_relative '../lib/lint/series'
18 require_relative '../lib/lint/symbols'
19 
20 =begin
21 -> build_source
22    - copy upstream source
23    - generate tarball
24    - copy packaging source
25    - fiddle
26    - update changelog
27    - dpkg-buildpackage -S
28 -> sign_source [elevated - possibly part of build_source]
29    - debsign
30 -> build binary
31    - upload_source
32    - wait_for_launchpad
33    - download_logs
34 -> check_logs
35    - cmake
36    - lintian
37    - symbols
38    - qml-checker
39 -> update_symbols
40 =end
41 
42 class KCIBuilder
43   class CoverageError < Exception; end
44 
45   DPUTCONF = '/var/lib/jenkins/tooling/dput.cf'
46   KEYID = '6A0C5BF2'
47   Project = Struct.new(:series, :stability, :name)
48 
49   def self.testing=(testing)
50     @testing = testing
51   end
52 
53   def self.testing
54     @testing ||= false
55   end
56 
57   def self.build_and_publish(project, source)
58     Dir.chdir('build') do
59       # Upload likes to get stuck, so we do timeout control to prevent all of
60       # builder from getting stuck.
61       # We try to dput two times in a row giving it first 30 minutes and then
62       # 15 minutes to complete. If it didn't manage to upload after that we
63       # ignore the package and move on.
64       `cp -rf /var/lib/jenkins/.ssh /root/`
65       `chown -Rv root:root /root/.ssh`
66       if `ssh-keygen -F ppa.launchpad.net`.strip.empty?
67         `ssh-keyscan -H ppa.launchpad.net >> ~/.ssh/known_hosts`
68       end
69       dput = "dput -d -c #{DPUTCONF} #{@ppa} #{source.name}_#{source.build_version.tar}*.changes"
70       success = false
71       4.times do |count|
72         # Note: count starts at 0 ;)
73         if timeout_spawn(dput, 60 * (30.0 / (count + 1)))
74           puts "spawn timeout goody"
75           success = true
76           break
77         end
78         puts "spawn timout badly"
79         sleep(60) # Sleep for a minute
80       end
81       abort '\t\t !!!!!!!!!!!! dput failed two times !!!!!!!!!!!!' unless success
82       Dir.chdir('..') do # main dir
83         puts "before require"
84         require_relative 'source_publisher'
85         puts "after require; before new"
86         publisher = SourcePublisher.new(source.name, source.version, project.stability)
87         puts "after new"
88         abort 'PPA Build Failed' unless publisher.wait
89         # Write upload data to file, we perhaps want to do something outside the
90         # build container.
91         data = { name: source.name,
92                  version: source.version,
93                  type: project.stability }
94         File.write('source.json', JSON.generate(data))
95       end
96     end
97   end
98 
99   def self.run
100     ENV['GNUPGHOME'] = '/var/lib/jenkins/tooling/gnupg'
101 
102     $stdout = $stderr
103 
104     # get basename, distro series, unstable/stable
105     components = ARGV.fetch(0).split('_')
106     unless components.size == 3
107       abort 'Did not get a valid project identifier via ARGV0'
108     end
109     project = Project.new(components.fetch(0),
110                           components.fetch(1),
111                           components.fetch(2))
112 
113     @ppa = "ppa:kubuntu-ci/#{project.stability}"
114 
115     # PWD
116     abort 'Could not change dir to ARGV1' unless Dir.chdir(ARGV.fetch(1))
117     @workspace_path = ARGV.fetch(1)
118 
119     # Workaround for docker not having suidmaps. We run as root in the docker
120     # which will result in uid/gid of written things to be 0 rather than whatever
121     # jenkins has. So instead we have a fake jenkins user in the docker we can
122     # chmod to. This ultimately ensures that the owernship is using the uid of
123     # the host jenkins (equal to docker jenkins) such that we don't end up with
124     # stuff owned by others.
125     at_exit do
126       FileUtils.chown_R('jenkins', 'jenkins', @workspace_path, verbose: true)
127     end unless testing
128 
129     source = VcsSourceBuilder.new(release: project.series).run
130 
131     # Mangle dsc to not do ARM builds unless explicitly enabled.
132     # With hundreds of distinct sources on CI, building all of them on three
133     # architectures, of which one is not too commonly used, is extremly excessive.
134     # Instead, by default we only build on the common architectures with extra
135     # architectures needing to be enabled explicitly.
136     # To achieve this mangle the Architecture field in the control file.
137     # If it contains an uncommon arch -> remove it -> if it is empty now, abort
138     # If it contains any -> replace with !uncommon
139     # This is a cheapster hack implementation to avoid having to implement write
140     # support in Debian control.
141     begin
142       Debian::DSCArch.twiddle!('build/')
143     rescue Debian::DSCArch::Error => e
144       # NOTE: this can raise a number of errors and we want them all to be fatal
145       #  to prevent unhandled dscs or completely empty architectures from
146       #  getting uploaded.
147       abort e
148     end
149 
150     fail CoverageError, 'Testing disabled after arch twiddle' if testing
151 
152     # Sign
153     Dir.chdir('build/') do
154       changes = Dir.glob('*.changes')
155       abort "Expected only one changes file #{changes}" if changes.size != 1
156       unless system("debsign -k#{KEYID} #{changes[0]}")
157         abort 'Failed to sign the source.'
158       end
159     end
160 
161     build_and_publish(project, source)
162 
163     unless File.exist?('logs/i386.log')
164       puts "found no logs"
165       exit 0
166     end
167 
168     # We need discrete arrays of both logs and architectures they represent
169     # to make sure we process them in the correct order when updating symbols.
170     logs = []
171     architectures_with_log = []
172     Dir.chdir('logs/') do
173       # `gunzip *.log.gz`
174       Dir.glob('*.log').each do |log|
175         logs << "#{@workspace_path}/logs/#{log}"
176         architectures_with_log << File.basename(log, '.log')
177       end
178     end
179 
180     archindep = File.read('archindep').strip rescue 'amd64' # Get archindep from PPA script.
181     log_data = File.open("logs/#{archindep}.log").read
182 
183     updated_symbols = update_symbols(log_data, logs, architectures_with_log,
184                                      project, source)
185 
186     puts_cmake(log_data, source.name)
187     puts_list_missing(log_data)
188     puts_lintian(log_data, updated_symbols: updated_symbols)
189 
190     # Lint control file
191     results = []
192     results << Lint::Control.new('packaging').lint
193     results << Lint::Series.new('packaging').lint
194     results << Lint::Symbols.new('packaging').lint
195 
196     Lint::ResultLogger.new(results).log
197 
198     # TODO: this script currently does not impact the build results nor does it
199     # create parsable output
200     qmlsrcs = %w(
201       bluez-qt
202       breeeze
203       kactivities
204       kalgebra
205       kanagram
206       kate
207       kbreakout
208       kdeplasma-addons
209       kdeclarative
210       kinfocenter
211       koko
212       kscreen
213       ktp-desktop-applets
214       kwin
215       libkdegames
216       plasma-framework
217       plasma-desktop
218       plasma-mediacenter
219       plasma-nm
220       plasma-sdk
221       plasma-volume-control
222       print-manager
223       purpose
224       milou
225       muon
226     )
227     if !Dir.glob('source/**/*.qml').empty? && qmlsrcs.include?(project.name)
228       require_relative 'lib/qml_dependency_verifier'
229 
230       dep_verify = QMLDependencyVerifier.new
231       dep_verify.add_ppa
232       missing_modules = dep_verify.missing_modules
233       missing_modules.each do |package, modules|
234         puts_warning "#{package} has missing dependencies..."
235         modules.uniq! { |mod| { mod.identifier => mod.version } }
236         modules.each do |mod|
237           puts_info "  #{mod} not found."
238           puts_info '    looked for:'
239           mod.import_paths.each do |path|
240             puts_info "      - #{path}"
241           end
242         end
243       end
244     end
245   end
246 
247   # Upload
248   def self.timeout_spawn(cmd, timeout)
249     thread = Thread.new do
250       loop do
251         warn '---- ps aux ----'
252         warn `ps aux |grep dput`
253         warn '----'
254         sleep 60
255       end
256     end
257 
258     pid = Process.spawn(cmd, pgroup: true)
259     begin
260       puts "waitpid with timeout"
261       Timeout.timeout(timeout) do
262         Process.waitpid(pid, 0)
263         puts "return wait pid"
264         return ($?.exitstatus == 0)
265       end
266     rescue Timeout::Error
267       puts "timeout error"
268       Process.kill(15, -Process.getpgid(pid))
269       return false
270     end
271   ensure
272     thread.kill
273   end
274 
275   def self.segmentify(data, start_marker, end_marker)
276     warn 'Compatibility function segmentify called. BuildLogSegmenter should be' \
277          ' used instead.'
278     BuildLogSegmenter.segmentify(data, start_marker, end_marker)
279   end
280 
281   def self.puts_kci(type, str)
282     Lint::ResultLogger.puts_kci(type, str)
283   end
284 
285   def self.puts_error(str)
286     puts_kci('E', str)
287   end
288 
289   def self.puts_warning(str)
290     puts_kci('W', str)
291   end
292 
293   def self.puts_info(str)
294     puts_kci('I', str)
295   end
296 
297   def self.ignore?(array_of_ignores, thing_to_check)
298     array_of_ignores.each do |possibly_ignore|
299       if thing_to_check == possibly_ignore ||
300          thing_to_check.start_with?(possibly_ignore) ||
301          thing_to_check.start_with?(possibly_ignore.chomp('*'))
302         return true
303       end
304     end
305     false
306   end
307 
308   def self.puts_cmake(data, source_name)
309     # Bit of compat code. Very lovely... NOT
310     parser = CMakeParser.new(data)
311     unless parser.valid
312       puts_info 'CMakeParser could not parse the log correctly. Aborting parser.'
313       return
314     end
315     missing = parser.missing
316     warnings = parser.warnings
317     disabled_features = parser.disabled_features
318 
319     puts 'KCI::CMAKE'
320 
321     # Force an ignore on Qt5TextToSpeech, it's not released as of right now.
322     # 2015-03-05
323     ignore_missing = %w(Qt5TextToSpeech)
324     cmake_ignore_path = "#{@workspace_path}/packaging/debian/meta/cmake-ignore"
325     if File.exist?(cmake_ignore_path)
326       ignore_missing += File.read(cmake_ignore_path).strip.split("\n")
327     end
328 
329     missing.each do |dep|
330       next if ignore?(ignore_missing, dep)
331       puts_warning("Missing Dep: #{dep}")
332     end
333 
334     warnings.each do |warning|
335       puts_warning(warning)
336     end
337 
338     disabled_features.each do |disabled_feature|
339       next if ignore?(ignore_missing, disabled_feature)
340       puts_warning("Disabled Feature: #{disabled_feature}")
341     end
342   end
343 
344   def self.puts_list_missing(data)
345     data = segmentify(data,
346                       "=== Start list-missing\n",
347                       "=== End list-missing\n")
348     return if data.empty?
349     puts 'KCI::MISSING'
350     data.each do |line|
351       # Missing files are always considered errors
352       puts_error(line)
353     end
354   end
355 
356   def self.puts_lintian(data, updated_symbols: false)
357     data = segmentify(data, "=== Start lintian\n", "=== End lintian\n")
358     return if data.empty?
359     puts 'KCI::LINTIAN'
360     data.each do |line|
361       next if line.start_with?('warning: ') # random warnings from lintian itself
362 
363       # Package names can easily go beyond what shit can suck on, so gag it.
364       next if line.include?('source-package-component-has-long-file-name')
365       next if line.include?('package-has-long-file-name')
366 
367       # We really do not care about standards versions for now. They only ever get
368       # bumped by the pkg-kde team anyway.
369       next if line.include?('out-of-date-standards-version')
370       next if line.include?('newer-standards-version')
371 
372       # We package an enormous amount of GUI apps without manpages (in fact
373       # they arguably wouldn't even make sense what with being GUI apps). So
374       # ignore any and all manpage warnings to save Harald from having to override
375       # them in every single application repository.
376       next if line.include?('binary-without-manpage')
377 
378       # Fuck you
379       next if line.include?('debian-revision-should-not-be-zero')
380       next if line.include?('bad-distribution-in-changes-file wily')
381       next if line.include?('not-binnmuable-any-depends-all')
382 
383       # Do not print symbols warnings if we already auto-updated.
384       if updated_symbols &&
385          line.include?('symbols-file-contains-current-version-with-debian-revision')
386         next
387       end
388 
389       case line[0..1]
390       when 'W:'
391         puts_warning(line)
392         next
393       when 'E:'
394         puts_error(line)
395         next
396       end
397       puts_info(line)
398     end
399   end
400 
401   def self.update_symbols(log_data, logs, architectures_with_log, project,
402                           source)
403     updated_symbols = false
404     # FIXME: stability wtf
405     gensymbols_regex = %r{dpkg-gensymbols: warning: (.*)/symbols doesn't match completely debian/(.*).symbols}
406     if project.series == KCI.latest_series && log_data.match(gensymbols_regex)
407       puts 'KCI::SYMBOLS'
408       if log_data.include?('warning: some symbols or patterns disappeared in the symbols file')
409         puts_error('It would very much appear that symbols have been retracted')
410       else
411         match = log_data.match(/--- debian\/(.*).symbols/)
412         if match && match.size > 1
413           Dir.chdir('packaging') do
414             system('git config --global user.email "kubuntu-ci@lists.launchpad.net"')
415             system('git config --global user.name "Kubuntu CI"')
416             system('git config core.sparsecheckout')
417             system("git checkout -f remotes/packaging/kubuntu_#{project.stability}")
418             system('git branch -a')
419             system('git status')
420             system('git reset --hard')
421             captures = match.captures
422             captures.each do |lib_package|
423               puts "pkgkde-symbolshelper batchpatch -v #{source.build_version.base} -c #{architectures_with_log.join(',')} #{logs.join(' ')}"
424               system("pkgkde-symbolshelper batchpatch -v #{source.build_version.base} -c #{architectures_with_log.join(',')} #{logs.join(' ')}")
425               updated_symbols = ($? == 0)
426               puts_info("Auto-updated symbols of #{lib_package}")
427             end
428             # Username et al apparently is somehow coming from .git or something
429             # apparently
430             system('git status')
431             system('git --no-pager diff')
432             system('git commit -a -m "Automatic symbol update"')
433           end
434         else
435           puts_error('Failed to update symbols as the package name(s) could not be'\
436                      ' parsed.')
437         end
438       end
439     end
440     updated_symbols
441   end
442 end
443 
444 if __FILE__ == $PROGRAM_NAME
445   File.open('/etc/apt/apt.conf.d/apt-cacher', 'w') do |file|
446     file.puts('Acquire::http { Proxy "http://10.0.3.1:3142"; };')
447   end
448 
449   `apt-get install -y xz-utils dpkg-dev ruby dput debhelper pkg-kde-tools devscripts python-launchpadlib ubuntu-dev-tools git`
450 
451   KCIBuilder.run
452 end

Generated on 2015-10-08 11:49:59 +0200 with SimpleCov-RCov 0.2.3