My Very Own CI-server
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

298 lines
9.7 KiB

// package build
// There are n build workers, which are started at the start of mvoCI.
// They execute BuildWorker. When a new build object is spawned into the
// build queue / channel, it is fetched (from git repo), the buildscript
// is placed in the folder and executed with bash. Then the directory
// is zipped and later removed.
package build
import (
"os"
"sync"
"os/exec"
"time"
"strings"
"strconv"
"io/ioutil"
"mvoCI/core"
"mvoCI/hook"
"github.com/jinzhu/gorm"
)
type BuilderConfig struct {
cfg *core.Config;
db *gorm.DB;
};
var c BuilderConfig;
// thread identifiers to the worker threads
var Workers map[int]core.Thread;
// channel / build queue for communication with the build workers
var InChannel chan core.Build
type mockerBuilder struct {
strings.Builder
}
func (m* mockerBuilder) Write(p []byte) (n int, err error) {
m.append ( string(p) )
return 0,nil
}
func (m* mockerBuilder) append ( args... string) {
for _, str := range args {
m.WriteString ( str );
}
}
var mux sync.Mutex;
// sets up the worker threads in the given quantitiy
func SetupWorkerThreads ( cfg *core.Config, db* gorm.DB ) {
c.cfg = cfg
c.db = db
numThreads := cfg.ParallelBuilds
// cleanup build directory
os.RemoveAll ( c.cfg.Directory.Repo )
InChannel = make( chan core.Build, 50 )
Workers = make(map[int]core.Thread)
for i:=0; i < numThreads; i++ {
Workers[i] = core.Pthread_create ( BuildWorker )
}
// builds, which where started or enqueued before the server was shut down -> enqueue them again
// and try to (re)build
var b []core.Build
c.db.Model ( &core.Build{} ).Where ("status = ? OR status = ?", "enqueued", "started").Find ( &b );
for _, bs := range b {
// enqueue
c.db.Model (&bs).Related (&bs.Repository);
bs.Status = "enqueued";
c.db.Save ( &bs )
InChannel <- bs
}
}
// worker function: loop indefinitely and retrieve a new build from the build
// queue and build it.
func BuildWorker () {
core.Console.Log ( "Starting BuildWorker" )
var b core.Build
var cnt int
for true {
b = core.Build{}
b =<-InChannel
mux.Lock();
c.db.Model ( &core.Build{} ).Where ( "repository_id = ? AND status = 'started'", b.RepositoryID ).Count ( &cnt );
core.Console.Log ( "CNT: ", cnt );
// if there is another thread building the same repo, reject it and back off
if cnt > 0 {
core.Console.Log ("Rejecting Build, another one on the same repo is running");
InChannel <- b
mux.Unlock();
time.Sleep ( 5 * time.Second );
} else {
b.Status = "started";
b.StartedAt = time.Now()
c.db.Model( &core.Build{} ).Save ( &b );
mux.Unlock();
core.Console.Log("Starting Build on ", b.Repository.Name )
log, err := WorkerGoGet ( &b );
b.FinishedAt = time.Now()
if err != true {
log.append ( "> Build failed\n");
b.Log = log.String();
b.Status = "failed";
core.Console.Log("Finished Build on ", b.Repository.Name )
c.db.Save ( &b );
WorkerGoCleanup ( &b, log )
} else {
log.append ( "> Build successful\n");
b.Log = log.String();
b.Status = "finished"
c.db.Save ( &b );
}
}
//core.LogLn ( b.Log );
}
}
// starts building a build, i.e. clone the repository, get the correct branch
// and commit, and calling every line of the build-script (WorkerGoBuild)
// if this all succeeds: set the status to "finished", "failed" otherwiese.
func WorkerGoGet ( b *core.Build ) (*mockerBuilder, bool) {
// create repo directory
var log mockerBuilder
log.append( "> Started build of ", b.Repository.Name, "\n" )
err := os.Mkdir ( c.cfg.Directory.Repo, 0777 );
err = os.Mkdir ( c.cfg.Directory.Build, 0777 );
log.append ( "> Cloning repository\n" )
dir := c.cfg.Directory.Repo + dirNameFromRepo ( b.Repository )
out, err := exec.Command ("git", "clone", b.Repository.CloneUrl, dir ).CombinedOutput()
log.append( string(out) )
if err != nil {
log.append("> Could not clone the repository :/\nLog: ", err.Error())
return &log, false;
}
log.append ( "> Switching branch to ", b.Branch, "\n");
cmd := exec.Command ("git", "checkout", b.Branch )
cmd.Dir = dir;
out, err = cmd.CombinedOutput()
log.append ( string(out) )
if err != nil {
log.append("> Could not switch to branch: ", err.Error(), "\n")
return &log, false;
}
cmd = exec.Command ("git", "rev-parse", "HEAD" )
cmd.Dir = dir
out, err = cmd.CombinedOutput()
if err != nil {
log.append("> Could not find out the current commit SHA: ", err.Error(), "\n")
return &log, false;
}
log.append ("> HEAD is ", string(out), "\n" )
if len(b.CommitSha) > 0 && b.CommitSha != string(out) {
log.append ( "> Fetching Commit ", b.CommitSha, "\n");
cmd = exec.Command ("git", "checkout", b.CommitSha )
cmd.Dir = dir
out, err = cmd.CombinedOutput()
log.append ( string(out) )
if err != nil {
log.append("> Could fetch the appropriate commit :/", err.Error(), "\n")
return &log, false;
}
} else {
b.CommitSha = string(out)
log.append ("> Building HEAD\n" )
}
// read author and commit-message from the git repository
if b.CommitAuthor == "" || b.CommitMessage == "" {
cmd = exec.Command ( "git", "show", "--format=%cn|%s", "--no-abbrev-commit", "--no-notes", "--no-patch" );
cmd.Dir = dir;
out, err = cmd.CombinedOutput();
o := strings.Split ( string(out), "\n" );
o2 := strings.Split ( o[0], "|");
b.CommitAuthor = strings.TrimSpace ( o2[0] );
b.CommitMessage = strings.TrimSpace ( o2[1] );
}
return WorkerGoBuild ( b, &log )
}
// Build the repo by executing every line of the build script
func WorkerGoBuild ( b *core.Build, log *mockerBuilder ) (*mockerBuilder, bool) {
log.append ( "> Starting Build ... \n")
var ShellScript string
ShellScript = b.BuildScript.ShellScript
buildScript := []byte( "set -e\n" + strings.ReplaceAll ( ShellScript, "\r", "\n") )
dir := c.cfg.Directory.Repo + dirNameFromRepo ( b.Repository )
buildScriptName := dir + "/mvoBuild.sh"
err := ioutil.WriteFile ( buildScriptName, buildScript, 0777 )
if err != nil {
log.append ( "> Could not create build script\n")
return log, false
}
l, e := execBuildScript ( "mvoBuild.sh", dir, b.CommitSha )
log.append ( l )
if e != true {
return log, false
}
return WorkerGoZip ( b, log );
}
// Zips the repo directory with the builds
func WorkerGoZip ( b *core.Build, log *mockerBuilder ) (*mockerBuilder, bool) {
if b.Repository.KeepBuilds {
var ZipName string = dirNameFromRepo ( b.Repository ) + "_" +
strconv.Itoa(int(b.ID)) + ".tar." +
c.cfg.CompressionMethod;
ZipName = strings.ReplaceAll ( ZipName, " ", "_" )
var ZipFile string = "../" + c.cfg.Directory.Build + ZipName
var inFile string = dirNameFromRepo ( b.Repository )
os.RemoveAll ( c.cfg.Directory.Repo + inFile + "/.git" )
log.append ( "> Zipping directory ", inFile, " to ", ZipFile, "\n" )
cmd := exec.Command ( "tar", "-cavf", ZipFile, inFile )
cmd.Dir = c.cfg.Directory.Repo
out, err := cmd.CombinedOutput ()
log.append( string(out) )
if err != nil {
log.append ( "Zipping failed: ", err.Error(), "\n" )
return log, false
}
b.Zip = ZipName
if b.Api == "gitea" && b.Event == "release" {
// only gitea has release-api event implemented, atm. (2020-06-27)
return WorkerPushRelease ( b, log );
}
} else {
b.Zip = ""
log.append ("> Skipping zipping because of repo-local configuration");
}
return WorkerGoCleanup ( b, log )
}
func WorkerPushRelease ( b *core.Build, log *mockerBuilder ) (*mockerBuilder, bool ) {
switch b.Api {
case "gitea":
err := hook.GiteaPushRelease ( b, b.ApiUrl, c.cfg.Directory.Build + "/" + b.Zip, c.cfg, c.db );
if err != nil {
log.append ("Error pushing the artifact: ", err.Error() );
} else {
log.append ("Pushing artifact should have worked.");
}
}
return WorkerGoCleanup ( b, log );
}
// removes the repo directory
func WorkerGoCleanup ( b *core.Build, log *mockerBuilder ) (*mockerBuilder, bool) {
log.append ( "> Cleaning up build directory\n" )
var inFile string = c.cfg.Directory.Repo +
dirNameFromRepo ( b.Repository )
os.RemoveAll ( inFile );
return log, true;
}
// execute the shell build script
func execBuildScript ( file string, dir string, buildVersion string ) (string, bool) {
var l mockerBuilder;
l.append ( "> Executing build script '", file, "'\n" )
cmd := exec.Command ( "bash", file )
cmd.Env = os.Environ();
cmd.Env = append ( cmd.Env, "VERSION="+buildVersion );
cmd.Dir = dir;
out, err := cmd.CombinedOutput();
l.append ( string(out) )
if err != nil {
l.append ( "> Execution failed with error: ", err.Error(), "\n" )
return l.String(), false;
}
return l.String(), true;
}
// returns the name of the repository for usage as directory name
func dirNameFromRepo ( r core.Repository ) string {
return r.Name;
}