From: Erik Brakkee Date: Wed, 21 Sep 2022 19:05:44 +0000 (+0200) Subject: initial revision. X-Git-Url: http://wamblee.org/gitweb/?p=pipelinelib;a=commitdiff_plain;h=74604a8509961af8543622c9b6c03b1a2a9edff9 initial revision. --- diff --git a/resources/podtemplates/google.yaml b/resources/podtemplates/google.yaml new file mode 100644 index 0000000..b2e1c3e --- /dev/null +++ b/resources/podtemplates/google.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +spec: + containers: + - name: google + image: google/cloud-sdk + #imagePullPolicy: Always + # this or any command that is + # bascially a noop is required, this is so that you don't overwrite the + # entrypoint of the base container + command: ["tail", "-f", "/dev/null"] + resources: + requests: + memory: "512M" + cpu: "500m" + diff --git a/resources/podtemplates/jnlp.yaml b/resources/podtemplates/jnlp.yaml new file mode 100644 index 0000000..1e66b60 --- /dev/null +++ b/resources/podtemplates/jnlp.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Pod +spec: + containers: + - name: jnlp + resources: + requests: + memory: "600M" + cpu: "550m" diff --git a/resources/podtemplates/kaniko.yaml b/resources/podtemplates/kaniko.yaml new file mode 100644 index 0000000..96b8a99 --- /dev/null +++ b/resources/podtemplates/kaniko.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +spec: + containers: + - name: kaniko + image: gcr.io/kaniko-project/executor:debug + #imagePullPolicy: Always + command: + - /busybox/cat + tty: true + resources: + requests: + memory: "2048M" + cpu: "1000m" + ephemeral-storage: "5Gi" + diff --git a/resources/scripts/processtemplates.groovy b/resources/scripts/processtemplates.groovy new file mode 100644 index 0000000..0530481 --- /dev/null +++ b/resources/scripts/processtemplates.groovy @@ -0,0 +1,26 @@ +import groovy.io.FileType + +variables = System.getenv() + +File dir = new File(".") + +def files = [] +dir.eachFileRecurse FileType.FILES, { + if (it.name =~ /\.template$/ ) { + files << it + } +} + +println(variables) +for (file in files) { + println(file) + content = file.text + + def engine = new groovy.text.SimpleTemplateEngine() + content = engine.createTemplate(content).make(variables).toString() + print("new content:" + content) + target = new File(file.absolutePath.replaceAll('\\.template$', "")) + target.text = content +} + + diff --git a/src/org/wamblee/jenkins/pipelinelib/MyYaml.groovy b/src/org/wamblee/jenkins/pipelinelib/MyYaml.groovy new file mode 100644 index 0000000..733e9b7 --- /dev/null +++ b/src/org/wamblee/jenkins/pipelinelib/MyYaml.groovy @@ -0,0 +1,130 @@ +package org.wamblee.jenkins.pipelinelib + +@Grab('org.yaml:snakeyaml:1.25') + +import org.yaml.snakeyaml.Yaml +import org.yaml.snakeyaml.constructor.SafeConstructor + +/* + Original reference: + https://github.com/OndraZizka/yaml-merge/blob/master/src/main/java/org/cobbzilla/util/yml/YmlMerger.java + */ + +class MyYaml { + private final Yaml parser + + MyYaml() { + parser = new Yaml(new SafeConstructor()) + } + + String merge(List yamls) { + Map mergedResult = new LinkedHashMap(); + for (yaml in yamls) { + final Map yamlToMerge = parser.load(yaml) + // Merge into results map. + mergeStructures(mergedResult, yamlToMerge) + } + return parser.dump(mergedResult) + } + + private static Object addToMergedResult(Map mergedResult, String key, Object yamlValue) { + return mergedResult.put(key, yamlValue) + } + + private static IllegalArgumentException unknownValueType(String key, Object yamlValue) { + final String msg = "Cannot mergeYamlFiles element of unknown type: " + key + ": " + yamlValue.getClass().getName() + return new IllegalArgumentException(msg) + } + + private void mergeLists(Map mergedResult, String key, Object yamlValue) { + if (!(yamlValue instanceof List && mergedResult.get(key) instanceof List)) { + throw new IllegalArgumentException("Cannot mergeYamlFiles a list with a non-list: " + key) + } + + List originalList = (List) mergedResult.get(key) + + // original implementation + // originalList.addAll((List) yamlValue) + + // my implementation + // below is non-standard approach as I assume a key:value mapping called (name->value) to identify a Map + List yamlList = (List) yamlValue + Map originalCache = new LinkedHashMap<>() + String name + for (ori in originalList) { + if (ori instanceof Map) { + name = ori.get('name') + if (name) { + originalCache.put(name, ori) + } + } + } + + def merged + for (item in yamlList) { + merged = false + if (item instanceof Map) { + name = item.get('name') + if (name && originalCache.containsKey(name)) { + mergeStructures((Map) originalCache.get(name), (Map) item) + merged = true + } + } + if (!merged) { + originalList.add(item) + } + } + } + + private void mergeStructures(Map targetTree, Map sourceTree) { + if (sourceTree == null) return + + for (String key : sourceTree.keySet()) { + + Object yamlValue = sourceTree.get(key) + if (yamlValue == null) { + addToMergedResult(targetTree, key, yamlValue) + continue + } + + Object existingValue = targetTree.get(key); + if (existingValue != null) { + if (yamlValue instanceof Map) { + if (existingValue instanceof Map) { + mergeStructures((Map) existingValue, (Map) yamlValue); + } else if (existingValue instanceof String) { + throw new IllegalArgumentException("Cannot mergeYamlFiles complex element into a simple element: " + key) + } else { + throw unknownValueType(key, yamlValue) + } + } else if (yamlValue instanceof List) { + mergeLists(targetTree, key, yamlValue) + + } else if (yamlValue instanceof String + || yamlValue instanceof Boolean + || yamlValue instanceof Double + || yamlValue instanceof Integer) { + + addToMergedResult(targetTree, key, yamlValue) + + } else { + throw unknownValueType(key, yamlValue) + } + + } else { + if (yamlValue instanceof Map + || yamlValue instanceof List + || yamlValue instanceof String + || yamlValue instanceof Boolean + || yamlValue instanceof Integer + || yamlValue instanceof Double) { + + addToMergedResult(targetTree, key, yamlValue) + } else { + throw unknownValueType(key, yamlValue) + } + } + } + } +} + diff --git a/vars/agentsetup.groovy b/vars/agentsetup.groovy new file mode 100644 index 0000000..ffa1d36 --- /dev/null +++ b/vars/agentsetup.groovy @@ -0,0 +1,69 @@ +// inspired/copied from https://github.com/liejuntao001/jenkins-k8sagent-lib + +import org.wamblee.jenkins.pipelinelib.MyYaml + +// containers: comma-separated list of containers to include. The order of the containers i +// important. Each container in the list corresponds to a yaml file in the podtemplates resource +// resource directory. +// repo: docker repo, default is the repo configured in the CONTAINER_REGISTRY environment variable +// version: version to use, defaults to BRANCH_NAME +// label: label to use for the agent. Defaults to the stage name if the agent is configured within a stage, +// otherwise the job name is used. +// +// All of the arguments specified in the call to agentsetup are passed without change to the +// pod template files and can be accessed in the yaml file as ${name} where name is the +// argument name. +def call(Map args) { + def defaults = [ + version: env.BRANCH_NAME, + repo: env.CONTAINER_REGISTRY, + label: env.STAGE_NAME ? env.STAGE_NAME: env.JOB_NAME, + idleMinutes: 0, + defaultContainer: null // initialized to first container later on + ] + args.label = env.JOB_NAME + if (env.STAGE_NAME) { + args.label = args.label + "-" + env.STAGE_NAME + } + args = defaults << args + + // combine the configured application templates + + def containers = args.containers.split(',').toList() + // always include the jnlp container to increase resource requirements. + // Otherwise, the checkout will be slow because of the limite amount of cpu that is + // reserverd. + if (!args.defaultContainer) { + args.defaultContainer = containers[0] + } + containers = containers.plus(0, 'jnlp') + + args.label = args.label.toLowerCase().replaceAll("[^a-zA-Z0-9]", "-").replaceAll("-+", "-") + println("agentsetup: containers to include: " + containers) + println("agentsetup: label '${args.label}'") + + + // all args are available to the templates + Map template_vars = args + + def templates = [] + for (container in containers ) { + template = libraryResource 'podtemplates/' + container + '.yaml' + template = renderTemplate(template, template_vars) + templates.add(template) + } + + def myyaml = new MyYaml() + def final_template = myyaml.merge(templates) + + ret = [:] + ret.idleMinutes = args.idleMinutes + ret.label = args.label + ret.yaml = final_template + ret.defaultContainer = args.defaultContainer + + println('agentsetup: parameters returned' + ret); + + ret + +} diff --git a/vars/applicationYaml.groovy b/vars/applicationYaml.groovy new file mode 100644 index 0000000..ead38a5 --- /dev/null +++ b/vars/applicationYaml.groovy @@ -0,0 +1,22 @@ + +def call(Map args) { + def defaults = [ + version: env.BRANCH_NAME, + repo: 'europe-west3-docker.pkg.dev/prod-cobundu-datascience-eu/ds', + ] + args = defaults << args + """ +apiVersion: v1 +kind: Pod +spec: + containers: + - name: ${args.application} + image: ${args.repo}/${args.application}:${args.version} + imagePullPolicy: Always + command: ["tail", "-f", "/dev/null"] + resources: + requests: + memory: "512M" + cpu: "2000m" + """ +} diff --git a/vars/buildcontainer.groovy b/vars/buildcontainer.groovy new file mode 100644 index 0000000..69763ce --- /dev/null +++ b/vars/buildcontainer.groovy @@ -0,0 +1,28 @@ +// Builds a docker container. This requires the kaniko container to be included using agentsetup +// +// Mandatory arguments: +// - context: the relative path in the source repository to the directory where the docker file is located +// - container: the name of the container to build +// +// Optional arguments: +// - cache: Whether to use the cache for building containers. Defaults to true +// - dockerfile: Name of the docker file in the context directory. Defaults to Dockerfile +// - repo: Repository to publish container to. Defaults to the CONTAINER_REGISTRY environment variable +// - version: Container version to build. Default to the value of the BRANCH_NAME variable +// +def call(Map args) { + def defaults = [ + cache: true, + cachettl: "100000h", + dockerfile: 'Dockerfile', + repo: env.CONTAINER_REGISTRY, + version: env.BRANCH_NAME + ] + args = defaults << args + container('kaniko') { + sh """ + echo "Building container with settings: ${args}" + /kaniko/executor --dockerfile ${args.dockerfile} --cache=${args.cache} --cache-ttl=${args.cachettl} --context \$( pwd )/${args.context} --destination ${args.repo}/${args.container}:${args.version} + """ + } +} diff --git a/vars/echotest.groovy b/vars/echotest.groovy new file mode 100644 index 0000000..3b32f2e --- /dev/null +++ b/vars/echotest.groovy @@ -0,0 +1,7 @@ + + +def call(String name) { + echo "Hello, ${name}" +} + + diff --git a/vars/k8sagent.groovy b/vars/k8sagent.groovy new file mode 100644 index 0000000..1082731 --- /dev/null +++ b/vars/k8sagent.groovy @@ -0,0 +1,32 @@ + +def call(Map args) { + def defaults = [ + version: env.BRANCH_NAME, + repo: 'europe-west3-docker.pkg.dev/prod-cobundu-datascience-eu/ds', + ] + args = defaults << args + ret = [:] + ret["label"] = args.application + ":" + args.version + ret["yaml"] = """ +apiVersion: v1 +kind: Pod +spec: + containers: + - name: ${args.application} + image: ${args.repo}/${args.application}:${args.version} + imagePullPolicy: Always + command: ["tail", "-f", "/dev/null"] + resources: + requests: + memory: "512M" + cpu: "2000m" + - name: jnlp + resources: + requests: + memory: "600M" + cpu: "550m" + +""" + ret + +} diff --git a/vars/processresources.groovy b/vars/processresources.groovy new file mode 100644 index 0000000..742b734 --- /dev/null +++ b/vars/processresources.groovy @@ -0,0 +1,58 @@ +// inspired/copied from https://github.com/liejuntao001/jenkins-k8sagent-lib + +import org.wamblee.jenkins.pipelinelib.MyYaml + +// containers: comma-separated list of containers to include. The order of the containers i +// important. Each container in the list corresponds to a yaml file in the podtemplates resource +// resource directory. +// repo: docker repo, default is the repo configured in the CONTAINER_REGISTRY environment variable +// version: version to use, defaults to BRANCH_NAME +// label: label to use for the agent. Defaults to the stage name if the agent is configured within a stage, +// otherwise the job name is used. +// +// All of the arguments specified in the call to agentsetup are passed without change to the +// pod template files and can be accessed in the yaml file as ${name} where name is the +// argument name. +def call(Map args) { + def defaults = [ + version: env.BRANCH_NAME, + repo: env.CONTAINER_REGISTRY, + label: env.STAGE_NAME ? env.STAGE_NAME: env.JOB_NAME, + ] + if (!args) { + args = [:] + } + args.label = env.JOB_NAME + if (env.STAGE_NAME) { + args.label = args.label + "-" + env.STAGE_NAME + } + args = defaults << args + + // combine the configured application templates + args.label = args.label.toLowerCase().replaceAll("[^a-zA-Z0-9]", "-").replaceAll("-+", "-") + + + command = "" + + command += ''' + files="$( find . -name '*.template' )" + for file in $files + do + base="$( dirname "$file")/$( basename "$file" .template )" + cat "$file" ''' + + for (key in args.keySet()) { + command += """ | sed 's|\\\$${key}|${args[key]}|g' """ + } + + command += ''' > "$file.tmp" + mv "$file.tmp" "$base" + done + ''' + + println "processresources: ${args}" + println "processresources: $command" + + sh "$command" + +} diff --git a/vars/renderTemplate.groovy b/vars/renderTemplate.groovy new file mode 100644 index 0000000..bd8dc8c --- /dev/null +++ b/vars/renderTemplate.groovy @@ -0,0 +1,7 @@ +import groovy.text.StreamingTemplateEngine + +def call(input, variables) { + def engine = new StreamingTemplateEngine() + return engine.createTemplate(input).make(variables).toString() +} +