Skip to content

Commit 4d6583e

Browse files
zeyadtamimiproblame
zeyadtamimi
and
problame
authored
config: support for including files & directories (conf.d) (#856)
This PR allows for the distribution of zrepl job definitions across multiple YAML files that are included from the main config. ``` global: ... include: - ./zrepl.yml.d - /opt/zrepl.yml - ... ``` Refer to the docs changes for details. Co-authored-by: Christian Schwarz <[email protected]>
1 parent 3f75127 commit 4d6583e

6 files changed

Lines changed: 169 additions & 4 deletions

File tree

‎docs/configuration/overview.rst‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,45 @@ Config File Structure
3030
A zrepl configuration file is divided in to two main sections: ``global`` and ``jobs``.
3131
``global`` has sensible defaults. It is covered in :ref:`logging <logging>`, :ref:`monitoring <monitoring>` \& :ref:`miscellaneous <miscellaneous>`.
3232

33+
34+
``conf.d``: including config files
35+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
36+
37+
It is possible to distribute zrepl configurations over multiple YAML configuration
38+
files. This is achieved by using the `include` key which can only exist in the main
39+
configuration file.
40+
41+
The list of included paths must point to either individual YAML files or directories.
42+
If the path points to a directory then all YAML files with a `.yml` extention in the
43+
directory will be included.
44+
45+
.. code-block:: yaml
46+
47+
# /etc/zrepl/zrep.yml
48+
global: ...
49+
include:
50+
- ./jobs.d/
51+
- /opt/some_job.yml
52+
53+
# /etc/zrepl/jobs.d/backup.yml
54+
jobs:
55+
- name: backup
56+
type: push
57+
- ...
58+
59+
# /opt/some_job.yml
60+
jobs:
61+
- name: another_job
62+
...
63+
- ...
64+
65+
66+
67+
The paths are treated as absolute when starting with `/`, otherwise
68+
as a relative path to the main config file's parent directory.
69+
70+
Job names must be unique across all included configuration files.
71+
3372
.. _job-overview:
3473

3574
Jobs \& How They Work Together

‎internal/config/config.go‎

Lines changed: 68 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import (
44
"fmt"
55
"log/syslog"
66
"os"
7+
pathpkg "path"
8+
"path/filepath"
79
"time"
810

911
"github.com/pkg/errors"
@@ -22,8 +24,9 @@ const (
2224
)
2325

2426
type Config struct {
25-
Jobs []JobEnum `yaml:"jobs,optional"`
26-
Global *Global `yaml:"global,optional,fromdefaults"`
27+
Jobs []JobEnum `yaml:"jobs,optional"`
28+
Global *Global `yaml:"global,optional,fromdefaults"`
29+
Include []string `yaml:"include,optional"`
2730
}
2831

2932
func (c *Config) Job(name string) (*JobEnum, error) {
@@ -655,8 +658,9 @@ var ConfigFileDefaultLocations = []string{
655658
"/usr/local/etc/zrepl/zrepl.yml",
656659
}
657660

658-
func ParseConfig(path string) (i *Config, err error) {
661+
func ParseConfig(path string) (rootConfig *Config, err error) {
659662

663+
// Parse main configuration file
660664
if path == "" {
661665
// Try default locations
662666
for _, l := range ConfigFileDefaultLocations {
@@ -679,7 +683,67 @@ func ParseConfig(path string) (i *Config, err error) {
679683
return
680684
}
681685

682-
return ParseConfigBytes(bytes)
686+
rootConfig, err = ParseConfigBytes(bytes)
687+
if err != nil {
688+
return nil, err
689+
}
690+
691+
err = expandConfigInclude(path, rootConfig)
692+
if err != nil {
693+
return nil, err
694+
}
695+
696+
return rootConfig, err
697+
}
698+
699+
func expandConfigInclude(configPath string, config *Config) (err error) {
700+
var includeConfigPaths []string
701+
for _, path := range config.Include {
702+
if !pathpkg.IsAbs(configPath) {
703+
path = pathpkg.Join(pathpkg.Dir(configPath), path)
704+
}
705+
706+
stat, statErr := os.Stat(path)
707+
if statErr != nil {
708+
return errors.Wrapf(statErr, "stat path %q", path)
709+
}
710+
711+
if stat.Mode().IsDir() {
712+
directoryPaths, err := filepath.Glob(path + "/*.yml")
713+
if err != nil {
714+
return err
715+
}
716+
717+
includeConfigPaths = append(includeConfigPaths, directoryPaths...)
718+
} else if stat.Mode().IsRegular() {
719+
if extention := filepath.Ext(path); extention != ".yml" {
720+
return fmt.Errorf("include config files must end with `.yml`: %s", path)
721+
}
722+
includeConfigPaths = append(includeConfigPaths, path)
723+
} else {
724+
return fmt.Errorf("not a file or directory: %s", path)
725+
}
726+
}
727+
728+
for _, path := range includeConfigPaths {
729+
var bytes []byte
730+
if bytes, err = os.ReadFile(path); err != nil {
731+
return errors.Wrapf(err, "read file: %q", path)
732+
}
733+
734+
includedConfig, err := ParseConfigBytes(bytes)
735+
if err != nil {
736+
return err
737+
}
738+
739+
if len(includedConfig.Include) > 0 {
740+
return errors.Errorf("included configuration files must not include other files: %s", path)
741+
}
742+
743+
config.Jobs = append(config.Jobs, includedConfig.Jobs...)
744+
}
745+
746+
return nil
683747
}
684748

685749
func ParseConfigBytes(bytes []byte) (*Config, error) {

‎internal/config/config_include_test.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package config
2+
3+
import (
4+
"testing"
5+
6+
"github.com/kr/pretty"
7+
"github.com/stretchr/testify/require"
8+
)
9+
10+
func TestIncludeSingle(t *testing.T) {
11+
path := "./samples/include.yml"
12+
13+
t.Run(path, func(t *testing.T) {
14+
config, err := ParseConfig(path)
15+
if err != nil {
16+
t.Errorf("error parsing %s:\n%+v", path, err)
17+
}
18+
19+
require.NotNil(t, config)
20+
require.NotNil(t, config.Global)
21+
require.NotEmpty(t, config.Jobs)
22+
23+
t.Logf("file: %s", path)
24+
t.Log(pretty.Sprint(config))
25+
})
26+
}
27+
28+
func TestIncludeDirectory(t *testing.T) {
29+
path := "./samples/include_directory.yml"
30+
31+
t.Run(path, func(t *testing.T) {
32+
config, err := ParseConfig(path)
33+
if err != nil {
34+
t.Errorf("error parsing %s:\n%+v", path, err)
35+
}
36+
37+
require.NotNil(t, config)
38+
require.NotNil(t, config.Global)
39+
require.NotEmpty(t, config.Jobs)
40+
41+
t.Logf("file: %s", path)
42+
t.Log(pretty.Sprint(config))
43+
})
44+
}

‎internal/config/samples/include.d/snap.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
jobs:
2+
- name: snapjob
3+
type: snap
4+
filesystems: {
5+
"tank/frequently_changed<": true,
6+
}
7+
snapshotting:
8+
type: periodic
9+
interval: 2m
10+
prefix: zrepl_snapjob_
11+
pruning:
12+
keep:
13+
- type: last_n
14+
count: 60

‎internal/config/samples/include.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
include:
2+
- ./snap.yml

‎internal/config/samples/include_directory.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
include:
2+
- ./include.d

0 commit comments

Comments
 (0)