diff --git a/deploy/example-configmap.yaml b/deploy/example-configmap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..aae6b64031a677924f6551baa0436cbd71f76f23
--- /dev/null
+++ b/deploy/example-configmap.yaml
@@ -0,0 +1,24 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+ name: local-path-config
+ namespace: default
+data:
+ config.json: |-
+ {
+ "nodePathMap":[
+ {
+ "node":"yasker-lp-dev1",
+ "paths":["/opt", "/data1"]
+ },
+ {
+ "node":"yasker-lp-dev2",
+ "paths":["/opt"]
+ },
+ {
+ "node":"yasker-lp-dev3",
+ "paths":["/data3"]
+ }
+ ]
+ }
+
diff --git a/deploy/provisioner.yaml b/deploy/provisioner.yaml
index bbe34b0dcdb59f6e5f6f16d5cd01979bb388fd39..7e37cc0c5b110949dde6a383a448d810750cd0c5 100644
--- a/deploy/provisioner.yaml
+++ b/deploy/provisioner.yaml
@@ -52,6 +52,7 @@ spec:
labels:
app: local-path-provisioner
spec:
+ serviceAccountName: local-path-provisioner-service-account
containers:
- name: local-path-provisioner
image: yasker/local-path-provisioner:63df7c1
@@ -60,4 +61,12 @@ spec:
- local-path-provisioner
- --debug
- start
- serviceAccountName: local-path-provisioner-service-account
+ - --config
+ - /etc/config/config.json
+ volumeMounts:
+ - name: config-volume
+ mountPath: /etc/config/
+ volumes:
+ - name: config-volume
+ configMap:
+ name: local-path-config
diff --git a/main.go b/main.go
index dfa7b05414b97fe3aaaf0e264c5ac0400e766c1f..31a0024429855359669d26167dae7f0b3f468886 100644
--- a/main.go
+++ b/main.go
@@ -22,6 +22,7 @@ var (
FlagProvisionerName = "provisioner-name"
EnvProvisionerName = "PROVISIONER_NAME"
+ FlagConfigFile = "config"
)
func cmdNotFound(c *cli.Context, command string) {
@@ -48,10 +49,15 @@ func StartCmd() cli.Command {
Flags: []cli.Flag{
cli.StringFlag{
Name: FlagProvisionerName,
- Usage: "Specify provisioner name",
+ Usage: "Optional. Specify provisioner name.",
EnvVar: EnvProvisionerName,
Value: DefaultProvisionerName,
},
+ cli.StringFlag{
+ Name: FlagConfigFile,
+ Usage: "Required. Provisioner configuration file.",
+ Value: "",
+ },
},
Action: func(c *cli.Context) {
if err := startDaemon(c); err != nil {
@@ -84,7 +90,11 @@ func startDaemon(c *cli.Context) error {
if provisionerName == "" {
return fmt.Errorf("invalid empty provisioner name")
}
- provisioner := NewProvisioner(kubeClient)
+ configFile := c.String(FlagConfigFile)
+ provisioner, err := NewProvisioner(kubeClient, configFile)
+ if err != nil {
+ return err
+ }
pc := pvController.NewProvisionController(
kubeClient,
provisionerName,
diff --git a/provisioner.go b/provisioner.go
index 144e3a1b83a1e215ad602c0319f18ca67481a1d8..910bff91b8243adcf66e450d2944785be5ae0ecf 100644
--- a/provisioner.go
+++ b/provisioner.go
@@ -1,7 +1,9 @@
package main
import (
+ "encoding/json"
"fmt"
+ "os"
"path/filepath"
"time"
@@ -19,18 +21,55 @@ const (
)
var (
- DefaultPath = "/opt"
- CleanupCounts = 120
+ CleanupTimeoutCounts = 120
)
type LocalPathProvisioner struct {
kubeClient *clientset.Clientset
+ configFile string
}
-func NewProvisioner(kubeClient *clientset.Clientset) *LocalPathProvisioner {
+type NodePathMapData struct {
+ Node string `json:"node,omitempty"`
+ Paths []string `json:"paths,omitempty"`
+}
+
+type ConfigData struct {
+ NodePathMap []*NodePathMapData `json:"nodePathMap,omitempty"`
+}
+
+type NodePathMap struct {
+ Paths map[string]struct{}
+}
+
+type Config struct {
+ NodePathMap map[string]*NodePathMap
+}
+
+func (c *Config) getRandomPathOnNode(node string) (string, error) {
+ npMap := c.NodePathMap[node]
+ if npMap == nil {
+ return "", fmt.Errorf("config doesn't contain node %v", node)
+ }
+ paths := npMap.Paths
+ if len(paths) == 0 {
+ return "", fmt.Errorf("no local path available on node %v", node)
+ }
+ path := ""
+ for path = range paths {
+ break
+ }
+ return path, nil
+}
+
+func NewProvisioner(kubeClient *clientset.Clientset, configFile string) (*LocalPathProvisioner, error) {
+ if _, err := getConfig(configFile); err != nil {
+ return nil, errors.Wrapf(err, "invalidate config file %v", configFile)
+ }
return &LocalPathProvisioner{
kubeClient: kubeClient,
- }
+ configFile: configFile,
+ }, nil
}
func (p *LocalPathProvisioner) Provision(opts pvController.VolumeOptions) (*v1.PersistentVolume, error) {
@@ -47,10 +86,20 @@ func (p *LocalPathProvisioner) Provision(opts pvController.VolumeOptions) (*v1.P
if opts.SelectedNode == nil {
return nil, fmt.Errorf("configuration error, no node was specified")
}
+
+ cfg, err := getConfig(p.configFile)
+ if err != nil {
+ return nil, err
+ }
+
+ basePath, err := cfg.getRandomPathOnNode(node.Name)
+ if err != nil {
+ return nil, err
+ }
name := opts.PVName
- path := filepath.Join(DefaultPath, name)
+ path := filepath.Join(basePath, name)
- logrus.Infof("Created volume %v", name)
+ logrus.Infof("Created volume %v at %v:%v", name, node.Name, path)
fs := v1.PersistentVolumeFilesystem
hostPathType := v1.HostPathDirectoryOrCreate
@@ -159,9 +208,12 @@ func (p *LocalPathProvisioner) cleanupVolume(name, path, node string) (err error
if name == "" || path == "" || node == "" {
return fmt.Errorf("invalid empty name or path or node")
}
- //TODO match up with node prefixes, make sure it's one of them (and not `/`)
- if filepath.Clean(path) == "/" {
- return fmt.Errorf("not sure why you want to DESTROY the root directory")
+ path, err = filepath.Abs(path)
+ if err != nil {
+ return err
+ }
+ if path == "/" {
+ return fmt.Errorf("not sure why you want to DESTROY THE ROOT DIRECTORY")
}
hostPathType := v1.HostPathDirectoryOrCreate
cleanupPod := &v1.Pod{
@@ -170,6 +222,7 @@ func (p *LocalPathProvisioner) cleanupVolume(name, path, node string) (err error
},
Spec: v1.PodSpec{
RestartPolicy: v1.RestartPolicyNever,
+ NodeName: node,
Containers: []v1.Container{
{
Name: "local-path-cleanup",
@@ -213,7 +266,7 @@ func (p *LocalPathProvisioner) cleanupVolume(name, path, node string) (err error
}()
completed := false
- for i := 0; i < CleanupCounts; i++ {
+ for i := 0; i < CleanupTimeoutCounts; i++ {
if pod, err := p.kubeClient.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{}); err != nil {
return err
} else if pod.Status.Phase == v1.PodSucceeded {
@@ -223,9 +276,59 @@ func (p *LocalPathProvisioner) cleanupVolume(name, path, node string) (err error
time.Sleep(1 * time.Second)
}
if !completed {
- return fmt.Errorf("cleanup process timeout after %v seconds", CleanupCounts)
+ return fmt.Errorf("cleanup process timeout after %v seconds", CleanupTimeoutCounts)
}
- logrus.Infof("Volume %v has been cleaned up", name)
+ logrus.Infof("Volume %v has been cleaned up from %v:%v", name, node, path)
return nil
}
+
+func getConfig(configFile string) (cfg *Config, err error) {
+ defer func() {
+ err = errors.Wrapf(err, "fail to load config file %v", configFile)
+ }()
+ f, err := os.Open(configFile)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ var data ConfigData
+ if err := json.NewDecoder(f).Decode(&data); err != nil {
+ return nil, err
+ }
+ cfg, err = canonicalizeConfig(&data)
+ if err != nil {
+ return nil, err
+ }
+ return cfg, nil
+}
+
+func canonicalizeConfig(data *ConfigData) (cfg *Config, err error) {
+ defer func() {
+ err = errors.Wrapf(err, "config canonicalization failed")
+ }()
+ cfg = &Config{}
+ cfg.NodePathMap = map[string]*NodePathMap{}
+ for _, n := range data.NodePathMap {
+ if cfg.NodePathMap[n.Node] != nil {
+ return nil, fmt.Errorf("duplicate node %v", n.Node)
+ }
+ npMap := &NodePathMap{Paths: map[string]struct{}{}}
+ cfg.NodePathMap[n.Node] = npMap
+ for _, p := range n.Paths {
+ if p[0] != '/' {
+ return nil, fmt.Errorf("path must start with / for path %v on node %v", p, n.Node)
+ }
+ path, err := filepath.Abs(p)
+ if err != nil {
+ return nil, err
+ }
+ if _, ok := npMap.Paths[path]; ok {
+ return nil, fmt.Errorf("duplicate path %v on node %v", p, n.Node)
+ }
+ npMap.Paths[path] = struct{}{}
+ }
+ }
+ return cfg, nil
+}