Anyone successfully building logstash java plugins as artifacts via gitlab runners? I’ve attempted a multi-stage docker build with the following gitlab-ci.yml, from which fails with the error
A problem was found with the configuration of task ':downloadJRuby' (type 'Download').
- Type 'de.undercouch.gradle.tasks.download.Download' property '$1' specifies file '/builds/project1/logstash/versions.yml' which doesn't exist.
gitlab-ci.yml:
default:
image: gradle:8.14.3-jdk17 # Use a suitable Gradle/JDK image
default:
image: gradle:8.14.3-jdk17 # Use a suitable Gradle/JDK image
variables:
# Ensure the Gradle daemon is disabled for CI/CD
GRADLE_OPTS: "-Dorg.gradle.daemon=false"
# Define a specific path for the Gradle user home to cache dependencies
GRADLE_USER_HOME: "/build/logstash/.gradle"
LOGSTASH_REPO_URL: "https://github.com/elastic/logstash.git"
#GIT_STRATEGY: none
stages:
- build_logstash
- build_extension
build_logstash_job:
stage: build_logstash
before_script:
- |
if [ -d "logstash" ]; then
echo "Repository already exists, performing git fetch..."
cd logstash
git fetch origin 8.19
#git reset --hard origin/8.19 # Or your desired branch
#cd ..
else
echo "Repository not found, performing git clone..."
git clone $LOGSTASH_REPO_URL --branch 8.19 --single-branch
#cd logstash
#git checkout $CI_COMMIT_REF_NAME # Or your desired branch
#cd ..
fi
script:
# Clone the Logstash repository into a specific directory
- pwd
#- mkdir /project1 && cd /project1
- ls -la
#- git clone $LOGSTASH_REPO_URL --branch 8.19 --single-branch
#- cd logstash
# Build Logstash (example command, adjust as needed for Logstash's specific build)
- ./gradlew --build-cache assemble
- gradle
#- pwd
#- ls -la
#- echo "Logstash build complete."
artifacts:
#Store necessary build outputs to be passed to the next stage
paths:
- logstash/rubyUtils.gradle
- logstash/values.yml
- logstash/logstash-core/build/libs/ # Example path for generated JARs/WARs, adjust as needed
#expire_in: 1 day
cache:
key: "$CI_COMMIT_REF_SLUG" # Cache per branch or a global key for shared cache
paths:
#- ../../../project1/logstash/.gradle/wrapper
#- ../../../project1/logstash/.gradle/caches
- logstash
- logstash/build
- logstash/.gradle
policy: pull-push # Use pull-push to fetch existing cache and upload a new one
build_extension_job:
stage: build_extension
# This job automatically has access to artifacts from the 'build_logstash' stage
script:
- echo "Building the extension using artifacts from the Logstash build."
# The logstash_src directory will be present due to artifacts and cache
#- cd /builds/project1/logstash/logstash-schema-validation # Change to your extension's project directory (if applicable)
# Reference the Logstash build outputs in your extension's build logic
#- ./gradlew build -PlogstashBuildDir="/project1/logstash/logstash-core/build/libs" # Pass artifact location as a Gradle property
- pwd
- ls -la
#- ls /project1
- gradle gem
- echo "Extension build complete."
#rules:
# - if: $CI_COMMIT_TAG
artifacts:
untracked: false
when: on_success
access: all
expire_in: 30 days
paths:
- "logstash-filter-project1_schema_validation-4.1.2-java.gem"
cache:
key: "$CI_COMMIT_REF_SLUG"
policy: pull # Only pull the cache, as this is the final build stage
build.gradle:
import java.nio.file.Files
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING
buildscript {
repositories {
mavenCentral()
gradlePluginPortal()
}
//dependencies {
// classpath "org.yaml:snakeyaml:2.2"
//}
}
plugins {
id 'com.github.johnrengelman.shadow' version '8.1.1'
id 'java'
}
ext {
//snakeYamlVersion = '2.2'
shadowGradlePluginVersion = '8.1.1'
}
// load versions.yml from core before loading rubyUtils.gradle
//import org.yaml.snakeyaml.Yaml
//def versionsFile = file("./versions.yml")
//def versionsFile = file(LOGSTASH_CORE_PATH + "/../versions.yml") // will need to add this to artifact list
//gradle.ext.versions = new Yaml().load(versionsFile.text)
//hardcoded to fix relative path confusion
apply from: "./logstash/rubyUtils.gradle"
// ===========================================================================
// plugin info
// ===========================================================================
group "project1.validation"
version "${file("VERSION").text.trim()}"
description = "Project1 plugin that formats and validates in coming data."
pluginInfo.licenses = ['']
pluginInfo.longDescription = "This plugin will map/validate incoming data based on provided mapping/schema files."
pluginInfo.authors = ['user']
pluginInfo.email = ['']
pluginInfo.homepage = ""
pluginInfo.pluginType = "filter"
pluginInfo.pluginClass = "Project1SchemaValidation"
pluginInfo.pluginName = "project1_schema_validation"
// ===========================================================================
project.logger.debug("setup complete")
repositories {
mavenCentral()
gradlePluginPortal()
}
shadowJar {
archiveClassifier.set('')
}
dependencies {
implementation 'org.apache.commons:commons-lang3:3.18.0'
implementation 'org.apache.logging.log4j:log4j-api:2.17.1'
implementation 'org.apache.logging.log4j:log4j-core:2.17.1'
implementation 'com.fasterxml.jackson.datatype:jackson-datatype-joda:2.12.1'
implementation 'com.networknt:json-schema-validator:1.4.0'
implementation files("/builds/project1/logstash/logstash-core/build/libs/logstash-core.jar")
}
clean {
delete "${projectDir}/Gemfile"
delete "${projectDir}/" + pluginInfo.pluginFullName() + ".gemspec"
delete "${projectDir}/lib/"
delete "${projectDir}/vendor/"
new FileNameFinder().getFileNames(projectDir.toString(), pluginInfo.pluginFullName() + "-?.?.?.gem").each { filename ->
delete filename
}
}
tasks.withType(JavaCompile) {
options.encoding = 'UTF-8'
}
tasks.register("vendor"){
dependsOn shadowJar
project.logger.debug("vendor")
doLast {
String vendorPathPrefix = "vendor/jar-dependencies"
String projectGroupPath = project.group.replaceAll('\\.', '/')
File projectJarFile = file("${vendorPathPrefix}/${projectGroupPath}/${pluginInfo.pluginFullName()}/${project.version}/${pluginInfo.pluginFullName()}-${project.version}.jar")
projectJarFile.mkdirs()
Files.copy(file("$buildDir/libs/${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING)
validatePluginJar(projectJarFile, project.group)
}
}
tasks.register("generateRubySupportFiles") {
doLast {
generateRubySupportFilesForPlugin(project.description, project.group, version)
}
}
tasks.register("removeObsoleteJars") {
project.logger.debug("remove obs jars")
doLast {
new FileNameFinder().getFileNames(
projectDir.toString(),
"vendor/**/" + pluginInfo.pluginFullName() + "*.jar",
"vendor/**/" + pluginInfo.pluginFullName() + "-" + version + ".jar").each { f ->
delete f
}
}
}
tasks.register("gem"){
project.logger.debug("gem")
dependsOn([downloadAndInstallJRuby, removeObsoleteJars, vendor, generateRubySupportFiles])
doLast {
buildGem(projectDir, buildDir, pluginInfo.pluginFullName() + ".gemspec")
}
}
Remaining build files follow the logstash java filter example in the docs. Any ideas?