Dockerize Custom Beat

Elastic Stack Beats
1

Hi :smiley:

I created a custom beat thanks to your Developer Guide , and would like to package it into a Docker Image. However, I did not find any information regarding that.

Are there any instructions on how to do that or even a simple command I could run?

Best,
Florian

2

Hey @Dev-Flo,

Try with mage package :slightly_smiling_face:

3

Thank you for your quick reply.

I ran mage -v package. But it seems to be stuck in a loop. I get the following output:

    2020/06/16 11:51:43 Found Elastic Beats dir at C:\Users\florianno\go\src\github.com\your-github-name\heartbeat\vendor\github.com\elastic\beats
    Running target: Package
    community_beat package spec loaded from [C:\Users\florianno\go\src\github.com\your-github-name\heartbeat\vendor\github.com\elastic\beats\dev-tools\packaging\packages.yml]
    Running dependency: Update
    exec: make update
    C:\Users\florianno\go\bin\mage.exe
    2020/06/16 11:52:23 Found Elastic Beats dir at C:\Users\florianno\go\src\github.com\your-github-name\heartbeat\vendor\github.com\elastic\beats
    Running target: Update
    exec: make update
    C:\Users\florianno\go\bin\mage.exe
    2020/06/16 11:53:02 Found Elastic Beats dir at C:\Users\florianno\go\src\github.com\your-github-name\heartbeat\vendor\github.com\elastic\beats
    Running target: Update
    exec: make update

Its keeps on repeating and never stops.

4

This is the package.yml (I did not change it)


# This file contains the package specifications for both Community Beats and
# Official Beats. The shared section contains YAML anchors that are used to
# define common parts of the package in order to not repeat ourselves.

shared:
  - &common
    name: '{{.BeatName}}'
    service_name: '{{.BeatServiceName}}'
    os: '{{.GOOS}}'
    arch: '{{.PackageArch}}'
    vendor: '{{.BeatVendor}}'
    version: '{{ beat_version }}'
    license: '{{.BeatLicense}}'
    url: '{{.BeatURL}}'
    description: '{{.BeatDescription}}'

  # Deb/RPM spec for community beats.
  - &deb_rpm_spec
    <<: *common
    post_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/systemd-daemon-reload.sh'
    files:
      /usr/share/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
        source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
        mode: 0755
      /etc/{{.BeatName}}/fields.yml:
        source: fields.yml
        mode: 0644
      /usr/share/{{.BeatName}}/LICENSE.txt:
        source: '{{ repo.RootDir }}/LICENSE.txt'
        mode: 0644
      /usr/share/{{.BeatName}}/NOTICE.txt:
        source: '{{ repo.RootDir }}/NOTICE.txt'
        mode: 0644
      /usr/share/{{.BeatName}}/README.md:
        template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl'
        mode: 0644
      /usr/share/{{.BeatName}}/.build_hash.txt:
        content: >
          {{ commit }}
        mode: 0644
      /etc/{{.BeatName}}/{{.BeatName}}.reference.yml:
        source: '{{.BeatName}}.reference.yml'
        mode: 0644
      /etc/{{.BeatName}}/{{.BeatName}}.yml:
        source: '{{.BeatName}}.yml'
        mode: 0600
        config: true
      /usr/share/{{.BeatName}}/kibana:
        source: _meta/kibana.generated
        mode: 0644
      /usr/share/{{.BeatName}}/bin/{{.BeatName}}-god:
        source: build/golang-crossbuild/god-{{.GOOS}}-{{.Platform.Arch}}
        mode: 0755
      /usr/bin/{{.BeatName}}:
        template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/linux/beatname.sh.tmpl'
        mode: 0755
      /lib/systemd/system/{{.BeatServiceName}}.service:
        template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/linux/systemd.unit.tmpl'
        mode: 0644
      /etc/init.d/{{.BeatServiceName}}:
        template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/{{.PackageType}}/init.sh.tmpl'
        mode: 0755

  # MacOS pkg spec for community beats.
  - &macos_beat_pkg_spec
    <<: *common
    extra_vars:
      # OS X 10.8 Mountain Lion is the oldest supported by Go 1.10.
      # https://golang.org/doc/go1.10#ports
      min_supported_osx_version: 10.8
      identifier: 'co.{{.BeatVendor | tolower}}.beats.{{.BeatName}}'
      install_path: /Library/Application Support
    pre_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/darwin/scripts/preinstall.tmpl'
    post_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/darwin/scripts/postinstall.tmpl'
    files:
      /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
        source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
        mode: 0755
      /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/LICENSE.txt:
        source: '{{ repo.RootDir }}/LICENSE.txt'
        mode: 0644
      /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/NOTICE.txt:
        source: '{{ repo.RootDir }}/NOTICE.txt'
        mode: 0644
      /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/README.md:
        template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl'
        mode: 0644
      /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/.build_hash.txt:
        content: >
          {{ commit }}
        mode: 0644
      /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/{{.identifier}}.plist:
        template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl'
        mode: 0644
      /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/kibana:
        source: _meta/kibana.generated
        mode: 0644
      /etc/{{.BeatName}}/fields.yml:
        source: fields.yml
        mode: 0644
      /etc/{{.BeatName}}/{{.BeatName}}.reference.yml:
        source: '{{.BeatName}}.reference.yml'
        mode: 0644
      /etc/{{.BeatName}}/{{.BeatName}}.yml:
        source: '{{.BeatName}}.yml'
        mode: 0600
        config: true

  - &binary_files
    '{{.BeatName}}{{.BinaryExt}}':
      source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
      mode: 0755
    fields.yml:
      source: fields.yml
      mode: 0644
    LICENSE.txt:
      source: '{{ repo.RootDir }}/LICENSE.txt'
      mode: 0644
    NOTICE.txt:
      source: '{{ repo.RootDir }}/NOTICE.txt'
      mode: 0644
    README.md:
      template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl'
      mode: 0644
    .build_hash.txt:
      content: >
        {{ commit }}
      mode: 0644
    '{{.BeatName}}.reference.yml':
      source: '{{.BeatName}}.reference.yml'
      mode: 0644
    '{{.BeatName}}.yml':
      source: '{{.BeatName}}.yml'
      mode: 0600
      config: true
    kibana:
      source: _meta/kibana.generated
      mode: 0644

  # Binary package spec (tar.gz for linux/darwin) for community beats.
  - &binary_spec
    <<: *common
    files:
      <<: *binary_files

  # Binary package spec (zip for windows) for community beats.
  - &windows_binary_spec
    <<: *common
    files:
      <<: *binary_files
      install-service-{{.BeatName}}.ps1:
        template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/windows/install-service.ps1.tmpl'
        mode: 0755
      uninstall-service-{{.BeatName}}.ps1:
        template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/windows/uninstall-service.ps1.tmpl'
        mode: 0755

  - &docker_spec
    <<: *binary_spec
    extra_vars:
      from: 'centos:7'
      user: '{{ .BeatName }}'
      linux_capabilities: ''
    files:
      '{{.BeatName}}.yml':
        source: '{{.BeatName}}.docker.yml'
        mode: 0600
        config: true

  - &elastic_docker_spec
    extra_vars:
      repository: 'docker.elastic.co/beats'

  #
  # License modifiers for Apache 2.0
  #
  - &apache_license_for_binaries
    license: "ASL 2.0"
    files:
      LICENSE.txt:
        source: '{{ repo.RootDir }}/licenses/APACHE-LICENSE-2.0.txt'
        mode: 0644

  - &apache_license_for_deb_rpm
    license: "ASL 2.0"
    files:
      /usr/share/{{.BeatName}}/LICENSE.txt:
        source: '{{ repo.RootDir }}/licenses/APACHE-LICENSE-2.0.txt'
        mode: 0644

  - &apache_license_for_macos_pkg
    license: "ASL 2.0"
    files:
      /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/LICENSE.txt:
        source: '{{ repo.RootDir }}/licenses/APACHE-LICENSE-2.0.txt'
        mode: 0644

  #
  # License modifiers for the Elastic License
  #
  - &elastic_license_for_binaries
    license: "Elastic License"
    files:
      LICENSE.txt:
        source: '{{ repo.RootDir }}/licenses/ELASTIC-LICENSE.txt'
        mode: 0644

  - &elastic_license_for_deb_rpm
    license: "Elastic License"
    files:
      /usr/share/{{.BeatName}}/LICENSE.txt:
        source: '{{ repo.RootDir }}/licenses/ELASTIC-LICENSE.txt'
        mode: 0644

  - &elastic_license_for_macos_pkg
    license: "Elastic License"
    files:
      /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/LICENSE.txt:
        source: '{{ repo.RootDir }}/licenses/ELASTIC-LICENSE.txt'
        mode: 0644

# specs is a list of named packaging "flavors".
specs:
  # Community Beats
  community_beat:
    - os: windows
      types: [zip]
      spec:
        <<: *windows_binary_spec

    - os: darwin
      types: [tgz]
      spec:
        <<: *binary_spec

    - os: linux
      types: [tgz]
      spec:
        <<: *binary_spec

    - os: linux
      types: [deb, rpm]
      spec:
        <<: *deb_rpm_spec

    - os: linux
      types: [docker]
      spec:
        <<: *docker_spec

  # Elastic Beat with Apache License (OSS) and binary taken the current
  # directory.
  elastic_beat_oss:
    - os: windows
      types: [zip]
      spec:
        <<: *windows_binary_spec
        <<: *apache_license_for_binaries
        name: '{{.BeatName}}-oss'

    - os: darwin
      types: [tgz]
      spec:
        <<: *binary_spec
        <<: *apache_license_for_binaries
        name: '{{.BeatName}}-oss'

    - os: darwin
      types: [dmg]
      spec:
        <<: *macos_beat_pkg_spec
        <<: *apache_license_for_macos_pkg
        name: '{{.BeatName}}-oss'

    - os: linux
      types: [tgz]
      spec:
        <<: *binary_spec
        <<: *apache_license_for_binaries
        name: '{{.BeatName}}-oss'

    - os: linux
      types: [deb, rpm]
      spec:
        <<: *deb_rpm_spec
        <<: *apache_license_for_deb_rpm
        name: '{{.BeatName}}-oss'

    - os: linux
      types: [docker]
      spec:
        <<: *docker_spec
        <<: *elastic_docker_spec
        <<: *apache_license_for_binaries
        name: '{{.BeatName}}-oss'

  # Elastic Beat with Elastic License and binary taken the current directory.
  elastic_beat_xpack:
    ###
    # Elastic Licensed Packages
    ###
    - os: windows
      types: [zip]
      spec:
        <<: *windows_binary_spec
        <<: *elastic_license_for_binaries

    - os: darwin
      types: [tgz]
      spec:
        <<: *binary_spec
        <<: *elastic_license_for_binaries

    - os: darwin
      types: [dmg]
      spec:
        <<: *macos_beat_pkg_spec
        <<: *elastic_license_for_macos_pkg

    - os: linux
      types: [tgz]
      spec:
        <<: *binary_spec
        <<: *elastic_license_for_binaries

    - os: linux
      types: [deb, rpm]
      spec:
        <<: *deb_rpm_spec
        <<: *elastic_license_for_deb_rpm

    - os: linux
      types: [docker]
      spec:
        <<: *docker_spec
        <<: *elastic_docker_spec
        <<: *elastic_license_for_binaries

  # Elastic Beat with Elastic License and binary taken from the x-pack dir.
  elastic_beat_xpack_separate_binaries:
    ###
    # Elastic Licensed Packages
    ###
    - os: windows
      types: [zip]
      spec:
        <<: *windows_binary_spec
        <<: *elastic_license_for_binaries
        files:
          '{{.BeatName}}{{.BinaryExt}}':
            source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

    - os: darwin
      types: [tgz]
      spec:
        <<: *binary_spec
        <<: *elastic_license_for_binaries
        files:
          '{{.BeatName}}{{.BinaryExt}}':
            source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

    - os: darwin
      types: [dmg]
      spec:
        <<: *macos_beat_pkg_spec
        <<: *elastic_license_for_macos_pkg
        files:
          /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
            mode: 0755
            source: ../x-pack/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

    - os: linux
      types: [tgz]
      spec:
        <<: *binary_spec
        <<: *elastic_license_for_binaries
        files:
          '{{.BeatName}}{{.BinaryExt}}':
            source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

    - os: linux
      types: [deb, rpm]
      spec:
        <<: *deb_rpm_spec
        <<: *elastic_license_for_deb_rpm
        files:
          /usr/share/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
            source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

    - os: linux
      types: [docker]
      spec:
        <<: *docker_spec
        <<: *elastic_docker_spec
        <<: *elastic_license_for_binaries
        files:
          '{{.BeatName}}{{.BinaryExt}}':
            source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
5

This loop looks like a bug, but I cannot reproduce it :thinking:

Do you know what version of beats you used to create tour custom beat?

6

Maybe this is something related to make and Windows. Try to replace the implementation of the Update target in magefile.go with something like this that won't use make.

func Update() {
        mg.SerialDeps(Fields, Config)
}
7

I'm not quite sure which version of beats I used. I created it like 10 months ago. So, whatever the current version was back then I used.

Strangely, the magefile.go did not have an Update function. Adding the Update function you propsed did not help. It is still stuck in the loop.

FYI, here is my magefile.go without adding the Update function.

// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

// +build mage

package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/magefile/mage/mg"
	"github.com/magefile/mage/sh"
	"github.com/pkg/errors"
	"go.uber.org/multierr"

	devtools "github.com/elastic/beats/dev-tools/mage"
)

var (
	// Beats is a list of Beats to collect dashboards from.
	Beats = []string{
		"heartbeat",
		"journalbeat",
		"packetbeat",
		"winlogbeat",
		"x-pack/auditbeat",
		"x-pack/filebeat",
		"x-pack/metricbeat",
		"x-pack/functionbeat",
	}
)

// PackageBeatDashboards packages the dashboards from all Beats into a zip
// file. The dashboards must be generated first.
func PackageBeatDashboards() error {
	version, err := devtools.BeatQualifiedVersion()
	if err != nil {
		return err
	}

	spec := devtools.PackageSpec{
		Name:     "beats-dashboards",
		Version:  version,
		Snapshot: devtools.Snapshot,
		Files: map[string]devtools.PackageFile{
			".build_hash.txt": devtools.PackageFile{
				Content: "{{ commit }}\n",
			},
		},
		OutputFile: "build/distributions/dashboards/{{.Name}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}",
	}

	for _, beatDir := range Beats {
		// The generated dashboard content is moving in the build dir, but
		// not all projects have been updated so detect which dir to use.
		dashboardDir := filepath.Join(beatDir, "build/kibana")
		legacyDir := filepath.Join(beatDir, "_meta/kibana.generated")
		beatName := filepath.Base(beatDir)

		if _, err := os.Stat(dashboardDir); err == nil {
			spec.Files[beatName] = devtools.PackageFile{Source: dashboardDir}
		} else if _, err := os.Stat(legacyDir); err == nil {
			spec.Files[beatName] = devtools.PackageFile{Source: legacyDir}
		} else {
			return errors.Errorf("no dashboards found for %v", beatDir)
		}
	}

	return devtools.PackageZip(spec.Evaluate())
}

// Fmt formats code and adds license headers.
func Fmt() {
	mg.Deps(devtools.GoImports, devtools.PythonAutopep8)
	mg.Deps(addLicenseHeaders)
}

// addLicenseHeaders adds ASL2 headers to .go files outside of x-pack and
// add Elastic headers to .go files in x-pack.
func addLicenseHeaders() error {
	fmt.Println(">> fmt - go-licenser: Adding missing headers")

	if err := sh.Run("go", "get", devtools.GoLicenserImportPath); err != nil {
		return err
	}

	return multierr.Combine(
		sh.RunV("go-licenser", "-license", "ASL2", "-exclude", "x-pack"),
		sh.RunV("go-licenser", "-license", "Elastic", "x-pack"),
	)
}

// DumpVariables writes the template variables and values to stdout.
func DumpVariables() error {
	return devtools.DumpVariables()
}
8

When opening then magefile.go in vscode though the following error is displayed:

can't load package: package github.com/your-github-name/heartbeat/vendor/github.com/elastic/beats: build constraints exclude all Go files in C:\Users\florianno\go\src\github.com\your-github-name\heartbeat\vendor\github.com\elastic\beats

The error is in [1,1], so first line, first character.

However, this error does not prevent the beat from building successfully.

9

Oh, it can be that. There have been many changes and fixes since them, including the migration to go modules, that should help on maintenance of custom beats.

Would it be an option for you to re-create your custom beat with a recent version?

10

I can do that. I'll let you know if that fixes the problem.

11

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.