From 2d8cd4a4789ec0f83ca1f1aafe4185da3a1f96d7 Mon Sep 17 00:00:00 2001 From: David C W Brown Date: Mon, 23 Apr 2018 17:28:43 +0100 Subject: [PATCH 1/3] Update ReadMe.md --- ReadMe.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ReadMe.md b/ReadMe.md index 6a545bbc..9b614a2e 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -1,4 +1,4 @@ -[![Build status](http://brownsmeet.com/githubhook/vishaps-status.svg)](http://brownsmeet.com/log/) +[Build status](http://brownsmeet.com/githubhook/vishaps-status.svg) # Ѵishap Oberon From 2de6345b99a609e00a8622898929dd41b202d54c Mon Sep 17 00:00:00 2001 From: David C W Brown Date: Mon, 23 Apr 2018 17:29:05 +0100 Subject: [PATCH 2/3] Update ReadMe.md --- ReadMe.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ReadMe.md b/ReadMe.md index 9b614a2e..f43f8559 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -1,4 +1,4 @@ -[Build status](http://brownsmeet.com/githubhook/vishaps-status.svg) +![Build status](http://brownsmeet.com/githubhook/vishaps-status.svg) # Ѵishap Oberon From 175eafb9009a786d4d9c69313132e8fa0d06faf4 Mon Sep 17 00:00:00 2001 From: David C W Brown Date: Mon, 23 Apr 2018 18:24:41 +0100 Subject: [PATCH 3/3] New autobuild system --- src/tools/autobuild/build-oberon.sh | 34 ---- src/tools/autobuild/buildall.pl | 68 -------- src/tools/autobuild/makesvg.pl | 113 +++++++++++++ src/tools/autobuild/postpush.pl | 55 ------- src/tools/autobuild/report.pl | 192 ---------------------- src/tools/autobuild/response.html | 5 + src/tools/autobuild/runbuilds.pl | 203 ++++++++++++++++++++++++ src/tools/autobuild/server.pl | 154 ++++++++++++++++++ src/tools/autobuild/update-webserver.sh | 9 -- 9 files changed, 475 insertions(+), 358 deletions(-) delete mode 100644 src/tools/autobuild/build-oberon.sh delete mode 100644 src/tools/autobuild/buildall.pl create mode 100644 src/tools/autobuild/makesvg.pl delete mode 100644 src/tools/autobuild/postpush.pl delete mode 100644 src/tools/autobuild/report.pl create mode 100644 src/tools/autobuild/response.html create mode 100644 src/tools/autobuild/runbuilds.pl create mode 100755 src/tools/autobuild/server.pl delete mode 100644 src/tools/autobuild/update-webserver.sh diff --git a/src/tools/autobuild/build-oberon.sh b/src/tools/autobuild/build-oberon.sh deleted file mode 100644 index ca9a10ed..00000000 --- a/src/tools/autobuild/build-oberon.sh +++ /dev/null @@ -1,34 +0,0 @@ -# Build Oberon -# -# Args -# -# $1 - whether to use sudo -# $2 - build directory -# $3 - CC -# $4 - branch - -if test "$1" = "sudo"; then sudo=sudo; else sudo=""; fi - -echo "" -echo === build-oberon.sh: \$1="$1", \$2="$2", \$3="$3", \$4="$4", \$sudo="$sudo" === -echo "" - -set -x -cd $2 - -$sudo git reset --hard # Clear the staging area -$sudo git clean -dfx # Remove all untracked files -$sudo git pull # Update the local repository -$sudo git checkout -f $4 # Switch to requested branch - -export CC=$3 - -if test "$3" = "msc-x86"; then - cmd /c 'C:\Program Files (x86)\Microsoft Visual C++ Build Tools\vcbuildtools.bat' x86 '&&' cd 'c:\cygwin\home\dave\msc-x86\voc' '&&' make full -else -if test "$3" = "msc-x64"; then - cmd /c 'C:\Program Files (x86)\Microsoft Visual C++ Build Tools\vcbuildtools.bat' x64 '&&' cd 'c:\cygwin\home\dave\msc-x64\voc' '&&' make full -else - $sudo make full -fi -fi diff --git a/src/tools/autobuild/buildall.pl b/src/tools/autobuild/buildall.pl deleted file mode 100644 index f72c36b9..00000000 --- a/src/tools/autobuild/buildall.pl +++ /dev/null @@ -1,68 +0,0 @@ -#!perl -w -use strict; -use warnings; -use POSIX "strftime"; -use Cwd; - -my $branch = "master"; -if (defined($ARGV[0]) && ($ARGV[0] ne "")) {$branch = $ARGV[0]} - -my %machines = ( - "pi" => ['22', 'pi@pie', 'sh build-oberon.sh sudo projects/oberon/vishap/voc gcc ' . $branch], - "darwin" => ['22', 'dave@dcb', 'sh build-oberon.sh sudo projects/oberon/vishap/voc clang ' . $branch], - "cygwin32" => ['5932', 'dave@wax', 'sh build-oberon.sh n oberon/cygwin/voc gcc ' . $branch], - "cygwin64" => ['5932', 'dave@wax', 'sh start64.sh \'sh build-oberon.sh n oberon/cygwin/voc gcc ' . $branch . '\''], - "mingw32" => ['5932', 'dave@wax', 'sh build-oberon.sh n oberon/mingw/voc i686-w64-mingw32-gcc ' . $branch], - "mingw64" => ['5932', 'dave@wax', 'sh start64.sh \'sh build-oberon.sh n oberon/mingw/voc x86_64-w64-mingw32-gcc ' . $branch . '\''], - "android" => ['8022', 'root@and', 'sh build-oberon.sh n vishap/voc gcc ' . $branch], - "lub64" => ['22', 'dave@vim', 'sh build-oberon.sh sudo oberon/voc gcc ' . $branch], - "lub32" => ['22', 'dave@vim-lub32', 'sh build-oberon.sh sudo oberon/voc gcc ' . $branch], - "fed64" => ['22', 'dave@vim-fed64', 'sh build-oberon.sh sudo oberon/voc gcc ' . $branch], - "osu64" => ['22', 'dave@vim-osu64', 'sh build-oberon.sh sudo oberon/voc gcc ' . $branch], - "obs32" => ['22', 'dave@vim-obs32', 'sh build-oberon.sh n vishap/voc gcc ' . $branch], - "win32" => ['22', 'dave@vim-win64', 'sh build-oberon.sh n msc-x86/voc msc-x86 ' . $branch], - "win64" => ['22', 'dave@vim-win64', 'sh build-oberon.sh n msc-x64/voc msc-x64 ' . $branch], - "ce64" => ['5922', 'obe@www', 'sh build-oberon.sh sudo vishap/voc gcc ' . $branch], - "fb64" => ['22', 'root@oberon', 'sh build-oberon.sh n vishap/voc gcc ' . $branch] -); - -sub logged { - my ($cmd, $id) = @_; - my $child = fork; - if (not defined $child) {die "Fork failed.";} - if ($child) { - # parent process - print "Started $id, pid = $child\n"; - } else { - # child process - open(my $log, ">log/$id.log") // die "Could not create log file log/$id.log"; - print $log strftime("%Y/%m/%d %H.%M.%S ", localtime), "$id.log\n"; - print $log strftime("%H.%M.%S", localtime), "> $cmd\n"; - print $id, " ", strftime("%H.%M.%S", localtime), "> $cmd\n"; - open(my $pipe, "$cmd 2>&1 |") // die "Could not open pipe from command $cmd."; - while (<$pipe>) { - my $line = $_; - print $id, " ", strftime("%H.%M.%S", localtime), " ", $line; - print $log strftime("%H.%M.%S", localtime), " ", $line; - } - close($pipe); - print $log strftime("%H.%M.%S", localtime), " --- Make completed ---\n"; - close($log); - exit; - } -} - -unlink glob "log/*"; - -for my $machine (sort keys %machines) { - my ($port, $login, $cmd) = @{$machines{$machine}}; - my $cmd = "scp -P $port build-oberon.sh $login:build-oberon.sh &&" - . "ssh -p $port $login \"$cmd\""; - logged($cmd, $machine); -} - -system("perl report.pl $branch"); -while ((my $pid = wait) > 0) { - print "Child pid $pid completed.\n"; - system("perl report.pl $branch"); -} diff --git a/src/tools/autobuild/makesvg.pl b/src/tools/autobuild/makesvg.pl new file mode 100644 index 00000000..398d09cd --- /dev/null +++ b/src/tools/autobuild/makesvg.pl @@ -0,0 +1,113 @@ +#!perl -w +use strict; +use warnings; + + +my %BuildStatus = (); +my $Rows = 0; +opendir DIR, "logs"; +while (my $fn = readdir(DIR)) { + if ($fn =~ /^((.+)-(.+))\.state$/) { + my ($build, $branch, $id) = ($1, $2, $3); + open STATE, "); # date time os compiler model compiler-build library-build ssource-change binary-change tests + splice(@state, 2, 0, $branch); + $BuildStatus{$build} = \@state; + close STATE; + $Rows++; + } +} +closedir DIR; + +#for my $fn (sort keys %BuildStatus) { +# print "$fn:\n"; +# my @state = @{$BuildStatus{$fn}}; +# my $i = 0; +# for my $val (@state) { +# print " $i: $val\n"; +# $i++; +# } +#} + + +my $FontHeight = 12; +my $LineHeight = 16; + +sub svgtext { + my ($f, $x, $y, $colour, $msg) = @_; + if ($msg ne '') { + $y = ($y+1)*$LineHeight + $FontHeight*0.4; + print $f <<"--END--TEXT--"; +$msg +--END--TEXT-- + } +} + +sub ColourFor { + my ($str) = @_; + if ($str eq "Failed") {return "#e03030";} # red + if ($str eq "Changed") {return "#ff9d4d";} # amber + if ($str eq "Passed") {return "#5adb5a";} # green + if ($str eq "Built") {return "#5adb5a";} # green + return "#c0c0c0"; +} + + +my @ColWidths = (22, 81, 67, 60, 70, 60, 50, 60, 60, 80, 80, 64); +my @Columns = (0); +for my $width (@ColWidths) {push @Columns, $Columns[$#Columns] + $width} + +my $Width = $Columns[$#Columns]; +my $Height = ($Rows+2.2) * $LineHeight; + + + +open(my $svg, ">vishaps-status.svg") // die "Could not create vishaps-status.svg."; + +print $svg <<"--END--SVG--HEADER--"; + + + +--END--SVG--HEADER-- + + +svgtext($svg, $Columns[1], 0, "#e0e0e0", "Date"); +svgtext($svg, $Columns[2], 0, "#e0e0e0", "Time"); +svgtext($svg, $Columns[3], 0, "#e0e0e0", "Branch"); +svgtext($svg, $Columns[4], 0, "#e0e0e0", "OS"); +svgtext($svg, $Columns[5], 0, "#e0e0e0", "Compiler"); +svgtext($svg, $Columns[6], 0, "#e0e0e0", "Model"); +svgtext($svg, $Columns[7], 0, "#e0e0e0", "Oberon"); +svgtext($svg, $Columns[8], 0, "#e0e0e0", "Library"); +svgtext($svg, $Columns[9], 0, "#e0e0e0", "C Source"); +svgtext($svg, $Columns[10], 0, "#e0e0e0", "Assembler"); +svgtext($svg, $Columns[11], 0, "#e0e0e0", "Tests"); + + +my $Row = 1; +for my $build (sort keys %BuildStatus) { + my @state = @{$BuildStatus{$build}}; + + my $y = $Row*$LineHeight + $FontHeight*0.8; + my $h = $LineHeight * 0.9; + print $svg <<"--END--HIGHLIGHT--"; + + +--END--HIGHLIGHT-- + + my $column = 1; + for my $field (@state) { + svgtext($svg, $Columns[$column], $Row, ColourFor($field), $field); + $column++; + } + $Row++; + + print $svg "\n" +} + +print $svg "\n"; +close $svg; diff --git a/src/tools/autobuild/postpush.pl b/src/tools/autobuild/postpush.pl deleted file mode 100644 index 07397517..00000000 --- a/src/tools/autobuild/postpush.pl +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/perl -w - -use strict; -use warnings; -use POSIX "strftime"; - -use CGI qw(:standard escapeHTML); -use JSON; - - -sub writelog { - my ($msg) = @_; - - open(LOG, ">>/tmp/postpush.log") or die "Could not create postpush.log"; - flock(LOG, 2) or die "Could not lock postpush.log"; - print LOG sprintf("%s %s\n", strftime("%Y/%m/%d %H.%M.%S", localtime), $msg); - close(LOG); - system "id >> /tmp/postpush.log"; -} - -my $postdata = from_json(param('POSTDATA')); - -my $url = $postdata->{'repository'}->{'url'}; -my $ref = $postdata->{'ref'}; -my $name = $postdata->{'head_commit'}->{'author'}->{'name'}; -my $branch = $ref; $branch =~ s'^.*\/''; -my $repo = $url; $repo =~ s'^.*\/''; -my $modified = $postdata->{'head_commit'}->{'modified'}; - -my $buildneeded = 0; -for my $file (@{$modified}) { - if ($file !~ m/\.(md|svg)$/i) {$buildneeded = 1;} -} - -if ($buildneeded) { - writelog "Post push github web hook for repository $repo, branch $branch, name $name. Build required."; - - my $child = fork; - if (not defined $child) {die "Fork failed.";} - if ($child) { - writelog "Started buildall, pid = $child."; # parent process - } else { - close(STDIN); close(STDOUT); close(STDERR); # child process - exec 'perl buildall.pl ' . $branch . ' >/tmp/buildall.log'; - exit; - } -} else { - writelog "Post push github web hook for repository $repo, branch $branch, name $name. No build required."; -} - - -print header(), - start_html("Vishap Oberon github post push web hook."), - p("Repository $repo, branch $branch, name $name."), - end_html(); diff --git a/src/tools/autobuild/report.pl b/src/tools/autobuild/report.pl deleted file mode 100644 index 80c8965e..00000000 --- a/src/tools/autobuild/report.pl +++ /dev/null @@ -1,192 +0,0 @@ -#!perl -w -use strict; -use warnings; -use POSIX "strftime"; -use Cwd; - -my $branch = "master"; -if (defined($ARGV[0]) && ($ARGV[0] ne "")) {$branch = $ARGV[0]} - -print "--- Running build status report on branch $branch.\n"; - -my %status = (); - -my $fn; -my $date; -my $time; -my $os; -my $compiler; -my $datamodel; -my $compilerok; -my $libraryok; -my $sourcechange; -my $asmchange; -my $tests; -my $key; -my $ver; - -sub clearvars { - $time = ""; $branch = ""; $os = ""; $compiler = ""; - $datamodel = ""; $compilerok = ""; $libraryok = ""; $sourcechange = ""; - $asmchange = ""; $tests = ""; $key = ""; $ver = ""; -} - -sub logstatus { - my ($fn) = @_; - if ($compiler ne "") { - $status{"$os-$compiler-$datamodel"} = - [$fn, $date, $time, $os, $compiler, $datamodel, $branch, $compilerok, $libraryok, $sourcechange, $asmchange, $tests]; - } - clearvars(); -} - -sub parselog { - ($fn) = @_; - clearvars(); - open(my $log, $fn) // die "Couldn't open build log $fn."; - $branch = "Build on $fn started"; - while (<$log>) { - s/\r//g; # Remove unwanted MS command prompt CR's. - - if (/^([0-9\/]+) ([0-9.]+) .+\.log$/) {$date = $1} - if (/^([0-9.]+) /) {$time = $1} - - #14.55.15 === build-oberon.sh: $1=sudo, $2=oberon/voc, $3=gcc, $4=master, $sudo=sudo === - if (/^[^ ]+ === build-oberon.sh: .* \$3=([^ ]+), \$4=([^ ]+),/) { - ($compiler, $branch) = ($1, $2); - } - - # 14.55.17 + sudo git checkout -f master - if (/^[^ ]+ .*git checkout -f ([^ ]+) *$/) { - $branch = $1; - } - - # 14.55.17 Configuration: 2.1.0 [2016/12/22] for gcc ILP32 on ubuntu - if (/^[^ ]+ Configuration: ([^ ]+) \[[0-9\/]+\] for ([^ ]+) ([^ ]+) on ([^ ]+)/) { - ($ver, $compiler, $datamodel, $os) = ($1, $2, $3, $4); - printf "--- Config for $fn: $1 for $2 $3 on $4.\n"; - } - - #if (/^[^ ]+ --- Cleaning branch ([^ ]+) ([^ ]+) ([^ ]+) ([^ ]+) ---$/) { - # ($branch, $os, $compiler, $datamodel) = ($1, $2, $3, $4, $5); - #} - - if (/^([0-9.]+) --- Compiler build started ---$/) {$compilerok = "Started";} - if (/^([0-9.]+) --- Compiler build successfull ---$/) {$compilerok = "Built";} - - if (/^([0-9.]+) --- Library build started ---$/) {$libraryok = "Started";} - if (/^([0-9.]+) --- Library build successfull ---$/) {$libraryok = "Built";} - - if (/^([0-9.]+) --- Generated c source files match bootstrap ---$/) {$sourcechange = "Unchanged";} - if (/^([0-9.]+) --- Generated c source files differ from bootstrap ---$/) {$sourcechange = "Changed";} - - if (/^([0-9.]+) --- Generated code unchanged ---$/) {if ($asmchange eq "") {$asmchange = "Unchanged"}} - if (/^([0-9.]+) --- Generated code changed ---$/) {$asmchange = "Changed"} - - if (/^([0-9.]+) --- Confidence tests started ---$/) {$tests = "Started";} - if (/^([0-9.]+) --- Confidence tests passed ---$/) {$tests = "Passed";} - - if (/^([0-9.]+) --- Make completed ---$/) { - # Go back and convert 'Started' status to 'Failed'. - if ($branch =~ m/^Build on/) {$branch = "Build on $fn failed to start.";} - if ($compilerok eq "Started") {$compilerok = "Failed";} - if ($libraryok eq "Started") {$libraryok = "Failed";} - if ($tests eq "Started") {$tests = "Failed";} - if ($compiler eq "msc") {$sourcechange = "n/a"; $tests = "n/a";} - } - } - close($log); - logstatus($fn); -} - -opendir DIR, "log" // die "Could not open log directory."; -my @logs = readdir DIR; -closedir DIR; - -for my $logname (sort @logs) { - $logname = "log/" . $logname; - #print "Consider $logname\n"; - if (-f $logname) {parselog($logname);} -} - -my $fontheight = 12; -my $lineheight = 15; - -sub svgtext { - my ($f, $x, $y, $colour, $msg) = @_; - print $f ''; - print $f $msg; - print $f "\n"; -} - -sub colourfor { - my ($str) = @_; - if ($str eq "Failed") {return "#e03030";} - if ($str eq "Changed") {return "#ff9d4d";} - if ($str eq "n/a") {return "#707070";} - return "#5adb5a"; -} - -my $rows = keys %status; - -my $width = 710; -my $height = ($rows+2.2) * $lineheight; - -open(my $svg, ">build-status.svg") // die "Could not create build-status.svg."; -print $svg '\n"; -print $svg '', "\n"; - -my $col1 = 20; -my $col2 = 97; -my $col3 = 160; -my $col4 = 220; -my $col5 = 280; -my $col6 = 330; -my $col7 = 380; -my $col8 = 440; -my $col9 = 490; -my $col10 = 570; -my $col11 = 650; - -svgtext($svg, $col1, 0, "#e0e0e0", "Date"); -svgtext($svg, $col3, 0, "#e0e0e0", "Branch"); -svgtext($svg, $col4, 0, "#e0e0e0", "Platform"); -svgtext($svg, $col7, 0, "#e0e0e0", "Compiler"); -svgtext($svg, $col8, 0, "#e0e0e0", "Library"); -svgtext($svg, $col9, 0, "#e0e0e0", "C Source"); -svgtext($svg, $col10, 0, "#e0e0e0", "Assembler"); -svgtext($svg, $col11, 0, "#e0e0e0", "Tests"); - -my $i=1; -for my $key (sort keys %status) { - my ($fn, $date, $time, $os, $compiler, $datamodel, $branch, $compilerok, $libraryok, - $sourcechange, $asmchange, $tests) = @{$status{$key}}; - print $svg ''; - svgtext($svg, $col1, $i, "#c0c0c0", $date); - svgtext($svg, $col2, $i, "#c0c0c0", $time); - svgtext($svg, $col3, $i, "#c0c0c0", $branch); - svgtext($svg, $col4, $i, "#c0c0c0", $os); - svgtext($svg, $col5, $i, "#c0c0c0", $compiler); - svgtext($svg, $col6, $i, "#c0c0c0", $datamodel); - svgtext($svg, $col7, $i, colourfor($compilerok), $compilerok); - svgtext($svg, $col8, $i, colourfor($libraryok), $libraryok); - svgtext($svg, $col9, $i, colourfor($sourcechange), $sourcechange); - svgtext($svg, $col10, $i, colourfor($asmchange), $asmchange); - svgtext($svg, $col11, $i, colourfor($tests), $tests); - print $svg ''; - $i++; -} - -print $svg "\n"; - -system 'chmod +r log/*'; -system 'scp build-status.svg dave@hub:/var/www'; -system 'scp log/* dave@hub:/var/www/log'; diff --git a/src/tools/autobuild/response.html b/src/tools/autobuild/response.html new file mode 100644 index 00000000..12acbc38 --- /dev/null +++ b/src/tools/autobuild/response.html @@ -0,0 +1,5 @@ + + +github webhook response +github webhook response + diff --git a/src/tools/autobuild/runbuilds.pl b/src/tools/autobuild/runbuilds.pl new file mode 100644 index 00000000..c0cb733a --- /dev/null +++ b/src/tools/autobuild/runbuilds.pl @@ -0,0 +1,203 @@ +#!perl -w +use strict; +use warnings; +use POSIX "strftime"; + +my $home = "/home/dave/vishap-build"; + +chdir $home; +mkdir "logs"; +#my $GlobalLog = *STDOUT; +open my $GlobalLog, ">$home/logs/runbuilds.log"; + +my $LogToScreenToo = 1; + + +# Find a build to run + +sub FindTriggeredBuild { + opendir (DIR, "$home") || die "$!"; + my $branch = undef; + while ((!defined $branch) && (my $fn = readdir DIR)) { + if ($fn =~ /^trigger-(\w+)/) {$branch = $1; unlink "$home/$fn"} + } + closedir DIR; + return $branch; +} + + + + +# Status information +# +# Build type: Datestamp Branch Platform Compiler Model +# Status: Compiler-build Library-build C-changed Assembler? Tests +# + +my %BuildStatus = (); + +sub WriteStatus {my ($timestamp, $id) = @_; + open STATE, ">logs/$id.state"; + print STATE "$timestamp "; + print STATE $BuildStatus{$id}->{kind}, ' '; + print STATE $BuildStatus{$id}->{compile}, ' '; + print STATE $BuildStatus{$id}->{library}, ' '; + print STATE $BuildStatus{$id}->{csource}, ' '; + print STATE $BuildStatus{$id}->{binary}, ' '; + print STATE $BuildStatus{$id}->{tests}, "\n"; + close STATE; +} + +sub SetStatus {my ($timestamp, $id, $section, $state) = @_; + if (!exists $BuildStatus{$id}) {$BuildStatus{$id} = { + kind => "- - -", compile => "Pending", library => "Pending", + csource => "Pending", binary => "Pending", tests => "Pending" + }} + $BuildStatus{$id}->{$section} = $state; + WriteStatus($timestamp, $id) +} + + +sub UpdateStatus {my ($timestamp, $id, $msg) = @_; + if ($msg =~ /^Configuration: ([^ ]+) \[[0-9\/]+\] for ([^ ]+) ([^ ]+) on ([^ ]+)/) { + SetStatus($timestamp, $id, 'kind', "$4 $2 $3") + } else { + if ($msg =~ /^--- (.*) ---$/) { + my $status = $1; + if ($status eq 'Build starts') {$id =~ /^.+?-(.+)$/; SetStatus($timestamp, $id, 'kind', "($1 rsync) -")} + elsif ($status eq 'Compiler build started') {SetStatus($timestamp, $id, 'compile', 'Busy')} + elsif ($status eq 'Compiler build successfull') {SetStatus($timestamp, $id, 'compile', 'Built')} + elsif ($status eq 'Library build started') {SetStatus($timestamp, $id, 'library', 'Busy')} + elsif ($status eq 'Library build successfull') {SetStatus($timestamp, $id, 'library', 'Built')} + elsif ($status eq 'Generated c source files match bootstrap') {SetStatus($timestamp, $id, 'csource', 'Unchanged')} + elsif ($status eq 'Generated c source files differ from bootstrap') {SetStatus($timestamp, $id, 'csource', 'Changed')} + elsif ($status eq 'Generated code unchanged') {SetStatus($timestamp, $id, 'binary', 'Unchanged')} + elsif ($status eq 'Generated code changed') {SetStatus($timestamp, $id, 'binary', 'Changed')} + elsif ($status eq 'Confidence tests started') {SetStatus($timestamp, $id, 'tests', 'Busy')} + elsif ($status eq 'Confidence tests passed') {SetStatus($timestamp, $id, 'tests', 'Passed')} + elsif ($status eq 'Build ends') { + my %status = %{$BuildStatus{$id}}; + foreach my $sec (keys %status) { + if ($status{$sec} eq 'Busy') {$BuildStatus{$id}->{$sec} = 'Failed'} + if ($status{$sec} eq 'Pending') {$BuildStatus{$id}->{$sec} = '-'} + } + WriteStatus($timestamp, $id) + } + } + } +} + + +sub Log {my ($log, $id, $msg) = @_; + my $timestamp = strftime("%Y/%m/%d %H.%M.%S", localtime); + $msg =~ s/[\r\n]*$//; # Remove trailing newline characters + UpdateStatus($timestamp, $id, $msg); + substr($timestamp,0,11) = ''; # Remove date part as not needed in logs + if ($LogToScreenToo) {print "($id) $timestamp $msg\n"} + print $GlobalLog "$timestamp ($id) $msg\n"; + if (defined $log) {print $log "$timestamp $msg\n"} +} + + +sub DoLogged {my ($log, $id, $cmd) = @_; + Log $log, $id, "Executing '$cmd'."; + open(my $pipe, "$cmd 2>&1 |") // die "Could not open pipe from command $cmd."; + while (<$pipe>) {Log $log, $id, $_} + close($pipe); +} + + + +sub SendFile {my ($log, $id, $dest, $port, $filename, $content) = @_; + Log $log, $id, "SendFile($dest,$port,$filename)"; + open PIPE, "|ssh -p $port $dest 'cat >$filename'"; + $content =~ s/\n/\r\n/g; # Unix to MS line ends. + print PIPE $content; + close PIPE; +} + + + +## my %machines = ( +## "cygwin32" => ['5932', 'dave@wax', 'sh build-oberon.sh n oberon/cygwin/voc gcc ' . $branch], +## "cygwin64" => ['5932', 'dave@wax', 'sh start64.sh \'sh build-oberon.sh n oberon/cygwin/voc gcc ' . $branch . '\''], +## "mingw32" => ['5932', 'dave@wax', 'sh build-oberon.sh n oberon/mingw/voc i686-w64-mingw32-gcc ' . $branch], +## "mingw64" => ['5932', 'dave@wax', 'sh start64.sh \'sh build-oberon.sh n oberon/mingw/voc x86_64-w64-mingw32-gcc ' . $branch . '\''], +## "ce64" => ['5922', 'obe@www', 'sh build-oberon.sh sudo vishap/voc gcc ' . $branch], +## ); + + + + +my @Builds = ( + ['pi', 'pi@pie', '22', '', 'cd vishaps/$id && make full'], + ['android', 'and', '8022', '', 'cd vishaps/$id && CC=gcc make full'], + ['lub32', 'vim-lub32', '22', '', 'cd vishaps/$id && make full'], + ['lub32cl', 'vim-lub32', '22', '', 'cd vishaps/$id && CC=clang make full'], + ['obs32', 'vim-obs32', '22', '', 'cd vishaps/$id && make full'], + ['cyg32', 'wax', '5932', '', 'cd vishaps/$id && make full'], + ['ming32', 'wax', '5932', '', 'cd vishaps/$id && CC=i686-w64-mingw32-gcc make full'], + ['cyg64', 'wax', '5932', '', 'sh start64.sh \'cd vishaps/$id && make full\''], + ['ming64', 'wax', '5932', '', 'sh start64.sh \'cd vishaps/$id && CC=x86_64-w64-mingw32-gcc make full\''], + ['lub64', 'vim', '22', '', 'cd vishaps/$id && make full'], + ['osu64', 'vim-osu64', '22', '', 'cd vishaps/$id && make full'], + ['fed64', 'vim-fed64', '22', '', 'cd vishaps/$id && make full'], + ['fbs64', 'githubhook', '22', '', 'cd vishaps/$id && make full'], + ['ce64', 'vim-ce64', '22', '', 'cd vishaps/$id && make full'], + ['darwin', 'dcb', '22', '', 'cd vishaps/$id && make full'], + ['win32', 'vim-win64', '22', 'x86', 'cmd /c x86.cmd'], + ['win64', 'vim-win64', '22', 'x64', 'cmd /c x64.cmd'] +); + +sub Prepare {my ($log, $id, $dest, $port, $preparation) = @_; + SendFile($log, $id, $dest, $port, "$preparation.cmd", <<"--END--MS--"); +call \"C:\\Program Files (x86)\\Microsoft Visual C++ Build Tools\\vcbuildtools.bat\" $preparation +cd %HOME%\\vishaps\\$id +make full +--END--MS-- +} + +sub BuildBranch {my ($branch) = @_; + Log undef, $branch, "$branch branch build triggered."; + +# # Obtain a clean clone of vishaps +# DoLogged undef, $branch, "rm -rf $home/voc"; +# DoLogged undef, $branch, "cd $home && git clone -b $branch --single-branch https://github.com/vishaps/voc"; + + # Start each build in turn + + unlink glob "$home/logs/$branch-*"; + for my $build (@Builds) { + my ($id, $dest, $port, $preparation, $command) = @$build; + my $rsynccompress = "-z"; if ($id eq "android") {$rsynccompress = "-zz"} + $id = "$branch-$id"; + my $child = fork; if (not defined $child) {die "Fork failed.";} + if ($child) {print "Opened process $child for build $id at $dest.\n"} + else { + # child process + my $log; + open $log, ">$home/logs/$id.log"; + Log $log, $id, "--- Build starts ---"; + Log $log, $id, strftime("%Y/%m/%d ", localtime) . "Build $id starting at $dest."; + DoLogged $log, $id, "ssh -p $port $dest mkdir -p vishaps/$id"; + DoLogged $log, $id, "rsync -r $rsynccompress --delete -e 'ssh -p $port' $home/voc/ $dest:vishaps/$id/"; + if ($preparation ne '') {Prepare($log, $id, $dest, $port, $preparation)} + $command =~ s/\$id /$id /g; + DoLogged $log, $id, "ssh -p $port $dest \"$command\""; + Log $log, $id, "Build $id for branch $branch at $dest completed."; + Log $log, $id, "--- Build ends ---"; + close $log; + exit; + } + } + + Log undef, $branch, "$branch branch: all builds started."; +} + + + +Log undef, 'runbuilds', strftime("%Y/%m/%d ", localtime) . "runbuilds starting."; +while (my $branch = FindTriggeredBuild()) {BuildBranch($branch)} +Log undef, 'runbuilds', "No more build triggers found, runbuilds complete."; +close $GlobalLog; + diff --git a/src/tools/autobuild/server.pl b/src/tools/autobuild/server.pl new file mode 100755 index 00000000..1016f78f --- /dev/null +++ b/src/tools/autobuild/server.pl @@ -0,0 +1,154 @@ +#!/usr/bin/perl +use strict; +use warnings; +use HTTP::Daemon; +use HTTP::Status; +use HTTP::Response; +use HTTP::Date qw(time2str); +use LWP::MediaTypes qw(guess_media_type); +use Digest::SHA qw(hmac_sha1_hex); +use Data::Dumper; +use JSON; +use sigtrap qw(die INT QUIT); + +my $home = "/usr/home/dave/vishap-build"; +my $criggleplop = "splurd crungle splonge."; + +$| = 1; + +my $PORT = 9000; +my $server = HTTP::Daemon->new(LocalPort => $PORT, ReuseAddr => 1); +die "Cannot setup server" unless $server; +print "[$$: Accepting clients at http://localhost:$PORT/]\n"; + +while (my $client = $server->accept) { + (my $pid = fork()) // die("Couldn't fork."); + if ($pid) {close $client; undef $client} else {ServeRequests($client); exit} +} +print "$$: server->accept failed. Exiting.\n"; +exit; + + +sub StartBuilds {my ($branch) = @_; + open TRIGGER, ">trigger-$branch"; print TRIGGER "trigger.\n"; close TRIGGER; + my $running = 0; + if (open PID, "; close PID} + if (!$running) { + my $child = fork; + if ($child) { + # This is the parent. We get to know the child PID, write it out. + print "Started build, pid $child.\n"; + open PID, ">pid"; print PID "$child.\n"; close PID; + } else { + # This is the child, we actually run all triggered builds. + exec "perl $home/runbuilds.pl >$home/runbuilds.log"; + exit; # Shouldn't get here + } + } +} + + +sub decodehook {my ($hook) = @_; + my %modifiedfiles = (); + my %commitauthors = (); + my $buildrequired = 0; + + my $commits = $hook->{commits}; + for my $commit (@$commits) { + my $committer = $commit->{committer}; + $commitauthors{$committer->{username}}++; + my $modified = $commit->{modified}; + for my $modfile (@$modified) { + $modifiedfiles{$modfile}++; + if ($modfile !~ m/\.(md|svg)$/i) {$buildrequired = 1;} + } + } + my $pusher = $hook->{pusher}; + my $repository = $hook->{repository}; + my $branch = $hook->{ref}; $branch =~ s'^.*\/''; + print "Repository: $repository->{name}, branch: $branch.\n"; + print "Commit authors: " . join(", ", keys %commitauthors) . ".\n"; + print "Pusher: " . $pusher->{name} . "\n"; + print "Files modified: " . join(", ", keys %modifiedfiles) . ".\n"; + print "Build " . ($buildrequired ? '' : 'not') . " required.\n"; + if ($buildrequired) {StartBuilds($branch)} +} + + +sub SendFile {my ($client, $file) = @_; + my $CRLF = "\r\n"; + local(*F); + sysopen(F, $file, 0); + binmode(F); + my($ct,$ce) = guess_media_type($file); + my($size,$mtime) = (stat $file)[7,9]; + $client->send_basic_header; + print $client "Content-Encoding: $ce$CRLF" if $ce; + print $client "Last-Modified: ", time2str($mtime), "$CRLF" if $mtime; + print $client "Accept-Ranges: bytes$CRLF"; + print $client "Content-Length: $size$CRLF" if $size; + print $client "Cache-Control: no-cache, no-store, must-revalidate, max-age=0$CRLF"; + print $client "Pragma: no-cache$CRLF"; + print $client "Expires: Wed, 11 Jan 1984 05:00:00 GMT$CRLF"; + print $client "Content-Type: $ct$CRLF"; + print $client $CRLF; + $client->send_file(\*F) unless $client->head_request; +} + + +sub VishapStatus {my ($client) = @_; + system "perl makesvg.pl"; # Construct an up-to-date status file + SendFile($client, "vishaps-status.svg"); +} + +sub LogFileResponse {my ($client, $path) = @_; + if (-f "logs/$path.log") { + SendFile($client, "logs/$path.log") + } else { + $client->send_error(RC_FORBIDDEN) + } +} + + +sub ServeRequests {my ($client) = @_; + $client->autoflush(1); + while (my $request = $client->get_request) { + if ($request->method eq 'GET') { + + my $uri = $request->uri; + my $host = $request->headers->{host}; + + print "URI requested: '$uri'.\n"; + print "Host: '$host'.\n"; + + $uri =~ s/^\///; + $uri =~ s/^githubhook\///; + + + if ($uri eq 'vishaps-status.svg') {VishapStatus($client)} + else {LogFileResponse($client, $uri)} + + + } elsif ($request->method eq 'POST') { + + $client->send_file_response("response.html"); + my $event = $request->headers->{'x-github-event'}; + my $githubsig = substr($request->headers->{'x-hub-signature'}, 5); + my $mysig = hmac_sha1_hex($request->content, $criggleplop); + print "Github event: $event, mysig $mysig, githubsig: $githubsig.\n"; + if (($event eq "push") && ($mysig eq $githubsig)) { + my $content = $request->content; + $content =~ s/%([0-9A-Fa-f]{2})/chr(hex($1))/eg; # Unescape %xx sequences. + decodehook(JSON::decode_json(substr($content,8))); + } + + } else { + + print "Request method $request->method forbidden.\n"; + $client->send_error(RC_FORBIDDEN) + } + } + $client->close; + undef $client; + #print "$$: no more requests, closing client.\n"; +} diff --git a/src/tools/autobuild/update-webserver.sh b/src/tools/autobuild/update-webserver.sh deleted file mode 100644 index 58a5e439..00000000 --- a/src/tools/autobuild/update-webserver.sh +++ /dev/null @@ -1,9 +0,0 @@ -# Push buildall and postpush to postpush server - -set -x -for f in *.pl build-oberon.sh; do - scp -P 5922 $f root@www:/var/lib/nethserver/ibay/githubhook/$f - ssh -p 5922 root@www "chmod +x /var/lib/nethserver/ibay/githubhook/$f" -done; -ssh -p 5922 root@www "ls -lap /var/lib/nethserver/ibay/githubhook" -