small refactoring to reduce os.exec calls

README.md

@@ -8,8 +8,8 @@ Originally created in [Flant](https://flant.com/) for internal needs & used for
 
 ## Features
 
-* Adding OpenVPN users (generating certificates for them);
+* Adding, deleting OpenVPN users (generating certificates for them);
-* Revoking/restoring users certificates;
+* Revoking/restoring/rotating users certificates;
 * Generating ready-to-user config files;
 * Providing metrics for Prometheus, including certificates expiration date, number of (connected/total) users, information about connected users;
 * (optionally) Specifying CCD (`client-config-dir`) for each user;
@@ -28,15 +28,12 @@ An example of dashboard made using ovpn-admin metrics:
 
 ## Installation
 
-### Disclaimer
-
-This tool uses external calls for `bash`, `coreutils` and `easy-rsa`, thus **Linux systems only are supported** at the moment.
-
 ### 1. Docker
 
 There is a ready-to-use [docker-compose.yaml](https://github.com/flant/ovpn-admin/blob/master/docker-compose.yaml), so you can just change/add values you need and start it with [start.sh](https://github.com/flant/ovpn-admin/blob/master/start.sh).
 
-Requirements. You need [Docker](https://docs.docker.com/get-docker/) and [docker-compose](https://docs.docker.com/compose/install/) installed.
+Requirements:
+You need [Docker](https://docs.docker.com/get-docker/) and [docker-compose](https://docs.docker.com/compose/install/) installed.
 
 Commands to execute:
 
@@ -45,6 +42,9 @@ git clone https://github.com/flant/ovpn-admin.git
 cd ovpn-admin
 ./start.sh
 ```
+#### 1.1
+Ready docker images available on [Docker Hub](https://hub.docker.com/r/flant/ovpn-admin/tags) 
+. Tags are simple: `$VERSION` or `latest` for ovpn-admin and `openvpn-$VERSION` or `openvpn-latest` for openvpn-server
 
 ### 2. Building from source
 
@@ -67,11 +67,17 @@ cd ovpn-admin
 
 ### 3. Prebuilt binary
 
-You can also download and use prebuilt binaries from the [releases](https://github.com/flant/ovpn-admin/releases) page — just choose a relevant tar.gz file.
+You can also download and use prebuilt binaries from the [releases](https://github.com/flant/ovpn-admin/releases/latest) page — just choose a relevant tar.gz file.
 
 
 ## Notes
-To use password authentication (the `--auth` flag) you have to install [openvpn-user](https://github.com/pashcovich/openvpn-user/releases). This tool should be available in your `$PATH` and its binary should be executable (`+x`).
+* this tool uses external calls for `bash`, `coreutils` and `easy-rsa`, thus **Linux systems only are supported** at the moment.
+* to enable additional password authentication provide `--auth` and `--auth.db="/etc/easyrsa/pki/users.db`" flags and install [openvpn-user](https://github.com/pashcovich/openvpn-user/releases/latest). This tool should be available in your `$PATH` and its binary should be executable (`+x`).
+* master-replica synchronization does not work with `--storage.backend=kubernetes.secrets` - **WIP**
+* additional password authentication does not work with `--storage.backend=kubernetes.secrets` -  **WIP**
+* if you use `--ccd` and `--ccd.path="/etc/openvpn/ccd"` abd plan to use static address setup for users do not forget to provide `--ovpn.network="172.16.100.0/24"` with valid openvpn-server network 
+* tested only with Openvpn-server versions 2.4 and 2.
+* status of users connections update every 28 second(*no need to ask why =)*)
 
 ## Usage
 
@@ -85,7 +91,7 @@ Flags:
   (or OVPN_LISTEN_HOST)
 
   --listen.port="8080"         port for ovpn-admin
-  (or OVPN_LISTEN_PROT)
+  (or OVPN_LISTEN_PORT)
 
   --role="master"              server role, master or slave
   (or OVPN_ROLE)
@@ -149,12 +155,6 @@ Flags:
 
   --auth.db="./easyrsa/pki/users.db"
   (or OVPN_AUTH_DB_PATH)      database path for password authorization
-
-  --debug                      enable debug mode
-  (or OVPN_DEBUG)
-
-  --verbose                    enable verbose mode
-  (or OVPN_VERBOSE)
   
   --log.level                  set log level: trace, debug, info, warn, error (default info)
   (or LOG_LEVEL)

helpers.go

@@ -1,11 +1,16 @@
 package main
 
 import (
+	"archive/tar"
+	"compress/gzip"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"strings"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -60,31 +65,80 @@ func fRead(path string) string {
 	return string(content)
 }
 
-func fCreate(path string) bool {
+func fCreate(path string) error {
 	var _, err = os.Stat(path)
 	if os.IsNotExist(err) {
 		var file, err = os.Create(path)
 		if err != nil {
 			log.Errorln(err)
-			return false
+			return err
 		}
 		defer file.Close()
 	}
-	return true
+	return nil
 }
 
-func fWrite(path, content string) {
+func fWrite(path, content string) error {
 	err := ioutil.WriteFile(path, []byte(content), 0644)
 	if err != nil {
 		log.Fatal(err)
 	}
+	return nil
 }
 
-func fDelete(path string) {
+func fDelete(path string) error {
 	err := os.Remove(path)
 	if err != nil {
 		log.Fatal(err)
 	}
+	return nil
+}
+
+func fCopy(src, dst string) error {
+	sfi, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+	if !sfi.Mode().IsRegular() {
+		// cannot copy non-regular files (e.g., directories, symlinks, devices, etc.)
+		return fmt.Errorf("fCopy: non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String())
+	}
+	dfi, err := os.Stat(dst)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+	} else {
+		if !(dfi.Mode().IsRegular()) {
+			return fmt.Errorf("fCopy: non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String())
+		}
+		if os.SameFile(sfi, dfi) {
+			return err
+		}
+	}
+	if err = os.Link(src, dst); err == nil {
+		return err
+	}
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		cerr := out.Close()
+		if err == nil {
+			err = cerr
+		}
+	}()
+	if _, err = io.Copy(out, in); err != nil {
+		return err
+	}
+	err = out.Sync()
+	return err
 }
 
 func fDownload(path, url string, basicAuth bool) error {
@@ -114,3 +168,124 @@ func fDownload(path, url string, basicAuth bool) error {
 
 	return nil
 }
+
+func createArchiveFromDir(dir, path string) error {
+
+	var files []string
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			log.Warn(err)
+			return err
+		}
+		if !info.IsDir() {
+			files = append(files, path)
+		}
+		return nil
+	})
+	if err != nil {
+		log.Warn(err)
+	}
+
+	out, err := os.Create(path)
+	if err != nil {
+		log.Errorf("Error writing archive %s: %s", path, err)
+		return err
+	}
+	defer out.Close()
+	gw := gzip.NewWriter(out)
+	defer gw.Close()
+	tw := tar.NewWriter(gw)
+	defer tw.Close()
+
+	// Iterate over files and add them to the tar archive
+	for _, filePath := range files {
+		file, err := os.Open(filePath)
+		if err != nil {
+			log.Warnf("Error writing archive %s: %s", path, err)
+			return err
+		}
+
+		// Get FileInfo about our file providing file size, mode, etc.
+		info, err := file.Stat()
+		if err != nil {
+			file.Close()
+			return err
+		}
+
+		// Create a tar Header from the FileInfo data
+		header, err := tar.FileInfoHeader(info, info.Name())
+		if err != nil {
+			file.Close()
+			return err
+		}
+
+		header.Name = strings.Replace(filePath, dir+"/", "", 1)
+
+		// Write file header to the tar archive
+		err = tw.WriteHeader(header)
+		if err != nil {
+			file.Close()
+			return err
+		}
+
+		// Copy file content to tar archive
+		_, err = io.Copy(tw, file)
+		if err != nil {
+			file.Close()
+			return err
+		}
+		file.Close()
+	}
+
+	return nil
+}
+
+func extractFromArchive(archive, path string) error {
+	// Open the file which will be written into the archive
+	file, err := os.Open(archive)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	// Write file header to the tar archive
+	uncompressedStream, err := gzip.NewReader(file)
+	if err != nil {
+		log.Fatal("extractFromArchive(): NewReader failed")
+	}
+
+	tarReader := tar.NewReader(uncompressedStream)
+
+	for true {
+		header, err := tarReader.Next()
+
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			log.Fatalf("extractFromArchive: Next() failed: %s", err.Error())
+		}
+
+		switch header.Typeflag {
+		case tar.TypeDir:
+			if err := os.Mkdir(path+"/"+header.Name, 0755); err != nil {
+				log.Fatalf("extractFromArchive: Mkdir() failed: %s", err.Error())
+			}
+		case tar.TypeReg:
+			outFile, err := os.Create(path + "/" + header.Name)
+			if err != nil {
+				log.Fatalf("extractFromArchive: Create() failed: %s", err.Error())
+			}
+			if _, err := io.Copy(outFile, tarReader); err != nil {
+				log.Fatalf("extractFromArchive: Copy() failed: %s", err.Error())
+			}
+			outFile.Close()
+
+		default:
+			log.Fatalf(
+				"extractFromArchive: uknown type: %s in %s", header.Typeflag, header.Name)
+		}
+	}
+	return nil
+}

kubernetes.go

@@ -7,9 +7,11 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
+	"github.com/google/uuid"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"strings"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -452,8 +454,8 @@ func (openVPNPKI *OpenVPNPKI) easyrsaRotate(commonName, newPassword string) (err
 	if err != nil {
 		log.Error(err)
 	}
-
+	uniqHash := strings.Replace(uuid.New().String(), "-", "", -1)
-	secret.Annotations["commonName"] = "REVOKED" + commonName
+	secret.Annotations["commonName"] = "REVOKED-" + commonName + "-" + uniqHash
 	secret.Labels["name"] = "REVOKED" + commonName
 	secret.Labels["revokedForever"] = "true"
 
@@ -490,9 +492,9 @@ func (openVPNPKI *OpenVPNPKI) easyrsaDelete(commonName string) (err error) {
 	if err != nil {
 		log.Error(err)
 	}
-
+	uniqHash := strings.Replace(uuid.New().String(), "-", "", -1)
-	secret.Annotations["commonName"] = "DELETED" + commonName
+	secret.Annotations["commonName"] = "REVOKED-" + commonName + "-" + uniqHash
-	secret.Labels["name"] = "DELETED" + commonName
+	secret.Labels["name"] = "REVOKED-" + commonName + "-" + uniqHash
 	secret.Labels["revokedForever"] = "true"
 
 	_, err = openVPNPKI.KubeClient.CoreV1().Secrets(namespace).Update(context.TODO(), secret, metav1.UpdateOptions{})

main.go

@@ -13,13 +13,13 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
-
 	"net"
 	"net/http"
 	"os"
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 	"text/template"
 	"time"
 
@@ -182,6 +182,7 @@ type OvpnAdmin struct {
 	templates              *packr.Box
 	modules                []string
 	mgmtStatusTimeFormat   string
+	createUserMutex        *sync.Mutex
 }
 
 type OpenvpnServer struct {
@@ -244,20 +245,20 @@ type clientStatus struct {
 }
 
 func (oAdmin *OvpnAdmin) userListHandler(w http.ResponseWriter, r *http.Request) {
-	log.Info(r.RemoteAddr, r.RequestURI)
+	log.Info(r.RemoteAddr, " ", r.RequestURI)
 	usersList, _ := json.Marshal(oAdmin.clients)
 	fmt.Fprintf(w, "%s", usersList)
 }
 
 func (oAdmin *OvpnAdmin) userStatisticHandler(w http.ResponseWriter, r *http.Request) {
-	log.Info(r.RemoteAddr, r.RequestURI)
+	log.Info(r.RemoteAddr, " ", r.RequestURI)
 	_ = r.ParseForm()
 	userStatistic, _ := json.Marshal(oAdmin.getUserStatistic(r.FormValue("username")))
 	fmt.Fprintf(w, "%s", userStatistic)
 }
 
 func (oAdmin *OvpnAdmin) userCreateHandler(w http.ResponseWriter, r *http.Request) {
-	log.Info(r.RemoteAddr, r.RequestURI)
+	log.Info(r.RemoteAddr, " ", r.RequestURI)
 	if oAdmin.role == "slave" {
 		http.Error(w, `{"status":"error"}`, http.StatusLocked)
 		return
@@ -275,7 +276,7 @@ func (oAdmin *OvpnAdmin) userCreateHandler(w http.ResponseWriter, r *http.Reques
 	}
 }
 func (oAdmin *OvpnAdmin) userRotateHandler(w http.ResponseWriter, r *http.Request) {
-	log.Info(r.RemoteAddr, r.RequestURI)
+	log.Info(r.RemoteAddr, " ", r.RequestURI)
 	if oAdmin.role == "slave" {
 		http.Error(w, `{"status":"error"}`, http.StatusLocked)
 		return
@@ -285,7 +286,7 @@ func (oAdmin *OvpnAdmin) userRotateHandler(w http.ResponseWriter, r *http.Reques
 }
 
 func (oAdmin *OvpnAdmin) userDeleteHandler(w http.ResponseWriter, r *http.Request) {
-	log.Info(r.RemoteAddr, r.RequestURI)
+	log.Info(r.RemoteAddr, " ", r.RequestURI)
 	if oAdmin.role == "slave" {
 		http.Error(w, `{"status":"error"}`, http.StatusLocked)
 		return
@@ -295,7 +296,7 @@ func (oAdmin *OvpnAdmin) userDeleteHandler(w http.ResponseWriter, r *http.Reques
 }
 
 func (oAdmin *OvpnAdmin) userRevokeHandler(w http.ResponseWriter, r *http.Request) {
-	log.Info(r.RemoteAddr, r.RequestURI)
+	log.Info(r.RemoteAddr, " ", r.RequestURI)
 	if oAdmin.role == "slave" {
 		http.Error(w, `{"status":"error"}`, http.StatusLocked)
 		return
@@ -305,7 +306,7 @@ func (oAdmin *OvpnAdmin) userRevokeHandler(w http.ResponseWriter, r *http.Reques
 }
 
 func (oAdmin *OvpnAdmin) userUnrevokeHandler(w http.ResponseWriter, r *http.Request) {
-	log.Info(r.RemoteAddr, r.RequestURI)
+	log.Info(r.RemoteAddr, " ", r.RequestURI)
 	if oAdmin.role == "slave" {
 		http.Error(w, `{"status":"error"}`, http.StatusLocked)
 		return
@@ -316,7 +317,7 @@ func (oAdmin *OvpnAdmin) userUnrevokeHandler(w http.ResponseWriter, r *http.Requ
 }
 
 func (oAdmin *OvpnAdmin) userChangePasswordHandler(w http.ResponseWriter, r *http.Request) {
-	log.Info(r.RemoteAddr, r.RequestURI)
+	log.Info(r.RemoteAddr, " ", r.RequestURI)
 	_ = r.ParseForm()
 	if *authByPassword {
 		passwordChanged, passwordChangeMessage := oAdmin.userChangePassword(r.FormValue("username"), r.FormValue("password"))
@@ -336,27 +337,27 @@ func (oAdmin *OvpnAdmin) userChangePasswordHandler(w http.ResponseWriter, r *htt
 }
 
 func (oAdmin *OvpnAdmin) userShowConfigHandler(w http.ResponseWriter, r *http.Request) {
-	log.Info(r.RemoteAddr, r.RequestURI)
+	log.Info(r.RemoteAddr, " ", r.RequestURI)
 	_ = r.ParseForm()
 	fmt.Fprintf(w, "%s", oAdmin.renderClientConfig(r.FormValue("username")))
 }
 
 func (oAdmin *OvpnAdmin) userDisconnectHandler(w http.ResponseWriter, r *http.Request) {
-	log.Info(r.RemoteAddr, r.RequestURI)
+	log.Info(r.RemoteAddr, " ", r.RequestURI)
 	_ = r.ParseForm()
 	// 	fmt.Fprintf(w, "%s", userDisconnect(r.FormValue("username")))
 	fmt.Fprintf(w, "%s", r.FormValue("username"))
 }
 
 func (oAdmin *OvpnAdmin) userShowCcdHandler(w http.ResponseWriter, r *http.Request) {
-	log.Info(r.RemoteAddr, r.RequestURI)
+	log.Info(r.RemoteAddr, " ", r.RequestURI)
 	_ = r.ParseForm()
 	ccd, _ := json.Marshal(oAdmin.getCcd(r.FormValue("username")))
 	fmt.Fprintf(w, "%s", ccd)
 }
 
 func (oAdmin *OvpnAdmin) userApplyCcdHandler(w http.ResponseWriter, r *http.Request) {
-	log.Info(r.RemoteAddr, r.RequestURI)
+	log.Info(r.RemoteAddr, " ", r.RequestURI)
 	if oAdmin.role == "slave" {
 		http.Error(w, `{"status":"error"}`, http.StatusLocked)
 		return
@@ -384,7 +385,7 @@ func (oAdmin *OvpnAdmin) userApplyCcdHandler(w http.ResponseWriter, r *http.Requ
 }
 
 func (oAdmin *OvpnAdmin) serverSettingsHandler(w http.ResponseWriter, r *http.Request) {
-	log.Info(r.RemoteAddr, r.RequestURI)
+	log.Info(r.RemoteAddr, " ", r.RequestURI)
 	enabledModules, enabledModulesErr := json.Marshal(oAdmin.modules)
 	if enabledModulesErr != nil {
 		log.Errorln(enabledModulesErr)
@@ -393,19 +394,23 @@ func (oAdmin *OvpnAdmin) serverSettingsHandler(w http.ResponseWriter, r *http.Re
 }
 
 func (oAdmin *OvpnAdmin) lastSyncTimeHandler(w http.ResponseWriter, r *http.Request) {
-	log.Debug(r.RemoteAddr, r.RequestURI)
+	log.Debug(r.RemoteAddr, " ", r.RequestURI)
 	fmt.Fprint(w, oAdmin.lastSyncTime)
 }
 
 func (oAdmin *OvpnAdmin) lastSuccessfulSyncTimeHandler(w http.ResponseWriter, r *http.Request) {
-	log.Debug(r.RemoteAddr, r.RequestURI)
+	log.Debug(r.RemoteAddr, " ", r.RequestURI)
 	fmt.Fprint(w, oAdmin.lastSuccessfulSyncTime)
 }
 
 func (oAdmin *OvpnAdmin) downloadCertsHandler(w http.ResponseWriter, r *http.Request) {
-	log.Info(r.RemoteAddr, r.RequestURI)
+	log.Info(r.RemoteAddr, " ", r.RequestURI)
 	if oAdmin.role == "slave" {
-		http.Error(w, `{"status":"error"}`, http.StatusLocked)
+		http.Error(w, `{"status":"error"}`, http.StatusBadRequest)
+		return
+	}
+	if *storageBackend == "kubernetes.secrets" {
+		http.Error(w, `{"status":"error"}`, http.StatusBadRequest)
 		return
 	}
 	_ = r.ParseForm()
@@ -422,9 +427,13 @@ func (oAdmin *OvpnAdmin) downloadCertsHandler(w http.ResponseWriter, r *http.Req
 }
 
 func (oAdmin *OvpnAdmin) downloadCcdHandler(w http.ResponseWriter, r *http.Request) {
-	log.Info(r.RemoteAddr, r.RequestURI)
+	log.Info(r.RemoteAddr, " ", r.RequestURI)
 	if oAdmin.role == "slave" {
-		http.Error(w, `{"status":"error"}`, http.StatusLocked)
+		http.Error(w, `{"status":"error"}`, http.StatusBadRequest)
+		return
+	}
+	if *storageBackend == "kubernetes.secrets" {
+		http.Error(w, `{"status":"error"}`, http.StatusBadRequest)
 		return
 	}
 	_ = r.ParseForm()
@@ -468,7 +477,7 @@ func main() {
 	ovpnAdmin.masterSyncToken = *masterSyncToken
 	ovpnAdmin.promRegistry = prometheus.NewRegistry()
 	ovpnAdmin.modules = []string{}
-
+	ovpnAdmin.createUserMutex = &sync.Mutex{}
 	ovpnAdmin.mgmtInterfaces = make(map[string]string)
 
 	for _, mgmtInterface := range *mgmtAddress {
@@ -659,7 +668,6 @@ func (oAdmin *OvpnAdmin) renderClientConfig(username string) string {
 		} else {
 			conf.Cert = fRead(*easyrsaDirPath + "/pki/issued/" + username + ".crt")
 			conf.Key = fRead(*easyrsaDirPath + "/pki/private/" + username + ".key")
-
 		}
 
 		conf.PasswdAuth = *authByPassword
@@ -726,22 +734,25 @@ func (oAdmin *OvpnAdmin) parseCcd(username string) Ccd {
 }
 
 func (oAdmin *OvpnAdmin) modifyCcd(ccd Ccd) (bool, string) {
-	ccdValid, ccdErr := validateCcd(ccd)
+	ccdValid, err := validateCcd(ccd)
-	if ccdErr != "" {
+	if err != "" {
-		return false, ccdErr
+		return false, err
 	}
 
 	if ccdValid {
 		t := oAdmin.getCcdTemplate()
 		var tmp bytes.Buffer
-		tplErr := t.Execute(&tmp, ccd)
+		err := t.Execute(&tmp, ccd)
-		if tplErr != nil {
+		if err != nil {
-			log.Error(tplErr)
+			log.Error(err)
 		}
 		if *storageBackend == "kubernetes.secrets" {
 			app.secretUpdateCcd(ccd.User, tmp.Bytes())
 		} else {
-			fWrite(*ccdDir+"/"+ccd.User, tmp.String())
+			err = fWrite(*ccdDir+"/"+ccd.User, tmp.String())
+			if err != nil {
+				log.Errorf("modifyCcd: fWrite(): %v", err)
+			}
 		}
 
 		return true, "ccd updated successfully"
@@ -850,7 +861,7 @@ func (oAdmin *OvpnAdmin) usersList() []OpenvpnClient {
 	apochNow := time.Now().Unix()
 
 	for _, line := range indexTxtParser(fRead(*indexTxtPath)) {
-		if line.Identity != "server" && !strings.Contains(line.Identity, "REVOKED") && !strings.Contains(line.Identity, "DELETED") {
+		if line.Identity != "server" && !strings.Contains(line.Identity, "REVOKED") {
 			totalCerts += 1
 			ovpnClient := OpenvpnClient{Identity: line.Identity, ExpirationDate: parseDateToString(indexTxtDateLayout, line.ExpirationDate, stringDateFormat)}
 			switch {
@@ -876,7 +887,7 @@ func (oAdmin *OvpnAdmin) usersList() []OpenvpnClient {
 			userConnected, userConnectedTo := isUserConnected(line.Identity, oAdmin.activeClients)
 			if userConnected {
 				ovpnClient.ConnectionStatus = "Connected"
-				for _ = range userConnectedTo {
+				for range userConnectedTo {
 					ovpnClient.Connections += 1
 					totalActiveConnections += 1
 				}
@@ -908,22 +919,25 @@ func (oAdmin *OvpnAdmin) usersList() []OpenvpnClient {
 func (oAdmin *OvpnAdmin) userCreate(username, password string) (bool, string) {
 	ucErr := fmt.Sprintf("User \"%s\" created", username)
 
+	oAdmin.createUserMutex.Lock()
+	defer oAdmin.createUserMutex.Unlock()
+
 	if checkUserExist(username) {
 		ucErr = fmt.Sprintf("User \"%s\" already exists\n", username)
-		log.Debugf("userCreate: %s", ucErr)
+		log.Debugf("userCreate: checkUserExist():  %s", ucErr)
 		return false, ucErr
 	}
 
 	if !validateUsername(username) {
 		ucErr = fmt.Sprintf("Username \"%s\" incorrect, you can use only %s\n", username, usernameRegexp)
-		log.Debugf("userCreate: %s", ucErr)
+		log.Debugf("userCreate: validateUsername(): %s", ucErr)
 		return false, ucErr
 	}
 
 	if *authByPassword {
 		if !validatePassword(password) {
 			ucErr = fmt.Sprintf("Password too short, password length must be greater or equal %d", passwordMinLength)
-			log.Debugf("userCreate: %s", ucErr)
+			log.Debugf("userCreate: authByPassword(): %s", ucErr)
 			return false, ucErr
 		}
 	}
@@ -934,7 +948,7 @@ func (oAdmin *OvpnAdmin) userCreate(username, password string) (bool, string) {
 			log.Error(err)
 		}
 	} else {
-		o := runBash(fmt.Sprintf("cd %s && easyrsa build-client-full %s nopass", *easyrsaDirPath, username))
+		o := runBash(fmt.Sprintf("cd %s && easyrsa build-client-full %s nopass 1>/dev/null", *easyrsaDirPath, username))
 		log.Debug(o)
 	}
 
@@ -954,24 +968,24 @@ func (oAdmin *OvpnAdmin) userChangePassword(username, password string) (bool, st
 
 	if checkUserExist(username) {
 		o := runBash(fmt.Sprintf("openvpn-user check --db.path %s --user %s | grep %s | wc -l", *authDatabase, username, username))
-		log.Info(o)
+		log.Debug(o)
 
 		if !validatePassword(password) {
 			ucpErr := fmt.Sprintf("Password for too short, password length must be greater or equal %d", passwordMinLength)
-			log.Debugf("userChangePassword: %s", ucpErr)
+			log.Warningf("userChangePassword: %s", ucpErr)
 			return false, ucpErr
 		}
 
 		if strings.TrimSpace(o) == "0" {
-			log.Info("Creating user in users.db")
+			log.Debug("Creating user in users.db")
 			o = runBash(fmt.Sprintf("openvpn-user create --db.path %s --user %s --password %s", *authDatabase, username, password))
-			log.Info(o)
+			log.Debug(o)
 		}
 
 		o = runBash(fmt.Sprintf("openvpn-user change-password --db.path %s --user %s --password %s", *authDatabase, username, password))
-		log.Info(o)
+		log.Debug(o)
 
-		log.Tracef("INFO: password for user %s was changed", username)
+		log.Infof("Password for user %s was changed", username)
 
 		return true, "Password changed"
 	}
@@ -991,7 +1005,6 @@ func (oAdmin *OvpnAdmin) getUserStatistic(username string) []clientStatus {
 
 func (oAdmin *OvpnAdmin) userRevoke(username string) string {
 	log.Infof("Revoke certificate for user %s", username)
-	var shellOut string
 	if checkUserExist(username) {
 		// check certificate valid flag 'V'
 		if *storageBackend == "kubernetes.secrets" {
@@ -1000,13 +1013,12 @@ func (oAdmin *OvpnAdmin) userRevoke(username string) string {
 				log.Error(err)
 			}
 		} else {
-			shellOut = runBash(fmt.Sprintf("date +%%Y-%%m-%%d\\ %%H:%%M:%%S && cd %s && echo yes | easyrsa revoke %s && easyrsa gen-crl", *easyrsaDirPath, username))
+			o := runBash(fmt.Sprintf("cd %s && echo yes | easyrsa revoke %s 1>/dev/null && easyrsa gen-crl 1>/dev/null", *easyrsaDirPath, username))
-			log.Debug(shellOut)
+			log.Debugln(o)
 		}
 
 		if *authByPassword {
-			shellOut = runBash(fmt.Sprintf("openvpn-user revoke --db-path %s --user %s", *authDatabase, username))
+			_ = runBash(fmt.Sprintf("openvpn-user revoke --db-path %s --user %s", *authDatabase, username))
-			log.Trace(shellOut)
 		}
 
 		crlFix()
@@ -1020,7 +1032,7 @@ func (oAdmin *OvpnAdmin) userRevoke(username string) string {
 		}
 
 		oAdmin.setState()
-		return fmt.Sprintln(shellOut)
+		return fmt.Sprintf("user \"%s\" revoked", username)
 	}
 	log.Infof("user \"%s\" not found", username)
 	return fmt.Sprintf("User \"%s\" not found", username)
@@ -1043,14 +1055,28 @@ func (oAdmin *OvpnAdmin) userUnrevoke(username string) string {
 						usersFromIndexTxt[i].Flag = "V"
 						usersFromIndexTxt[i].RevocationDate = ""
 
-						_ = runBash(fmt.Sprintf("cd %s && cp pki/revoked/certs_by_serial/%s.crt pki/issued/%s.crt", *easyrsaDirPath, usersFromIndexTxt[i].SerialNumber, username))
+						err := fCopy(fmt.Sprintf("%s/pki/revoked/certs_by_serial/%s.crt", *easyrsaDirPath, usersFromIndexTxt[i].SerialNumber), fmt.Sprintf("%s/pki/issued/%s.crt", *easyrsaDirPath, username))
-						_ = runBash(fmt.Sprintf("cd %s && cp pki/revoked/certs_by_serial/%s.crt pki/certs_by_serial/%s.pem", *easyrsaDirPath, usersFromIndexTxt[i].SerialNumber, usersFromIndexTxt[i].SerialNumber))
+						if err != nil {
-						_ = runBash(fmt.Sprintf("cd %s && cp pki/revoked/private_by_serial/%s.key pki/private/%s.key", *easyrsaDirPath, usersFromIndexTxt[i].SerialNumber, username))
+							log.Error(err)
-						_ = runBash(fmt.Sprintf("cd %s && cp pki/revoked/reqs_by_serial/%s.req pki/reqs/%s.req", *easyrsaDirPath, usersFromIndexTxt[i].SerialNumber, username))
+						}
+						err = fCopy(fmt.Sprintf("%s/pki/revoked/certs_by_serial/%s.crt", *easyrsaDirPath, usersFromIndexTxt[i].SerialNumber), fmt.Sprintf("%s/pki/certs_by_serial/%s.pem", *easyrsaDirPath, usersFromIndexTxt[i].SerialNumber))
+						if err != nil {
+							log.Error(err)
+						}
+						err = fCopy(fmt.Sprintf("%s/pki/revoked/private_by_serial/%s.key", *easyrsaDirPath, usersFromIndexTxt[i].SerialNumber), fmt.Sprintf("%s/pki/private/%s.key", *easyrsaDirPath, username))
+						if err != nil {
+							log.Error(err)
+						}
+						err = fCopy(fmt.Sprintf("%s/pki/revoked/reqs_by_serial/%s.req", *easyrsaDirPath, usersFromIndexTxt[i].SerialNumber), fmt.Sprintf("%s/pki/reqs/%s.req", *easyrsaDirPath, username))
+						if err != nil {
+							log.Error(err)
+						}
+						err = fWrite(*indexTxtPath, renderIndexTxt(usersFromIndexTxt))
+						if err != nil {
+							log.Error(err)
+						}
 
-						fWrite(*indexTxtPath, renderIndexTxt(usersFromIndexTxt))
+						_ = runBash(fmt.Sprintf("cd %s && easyrsa gen-crl 1>/dev/null", *easyrsaDirPath))
-
-						_ = runBash(fmt.Sprintf("cd %s && easyrsa gen-crl", *easyrsaDirPath))
 
 						if *authByPassword {
 							_ = runBash(fmt.Sprintf("openvpn-user restore --db-path %s --user %s", *authDatabase, username))
@@ -1062,7 +1088,10 @@ func (oAdmin *OvpnAdmin) userUnrevoke(username string) string {
 					}
 				}
 			}
-			fWrite(*indexTxtPath, renderIndexTxt(usersFromIndexTxt))
+			err := fWrite(*indexTxtPath, renderIndexTxt(usersFromIndexTxt))
+			if err != nil {
+				log.Error(err)
+			}
 			//fmt.Print(renderIndexTxt(usersFromIndexTxt))
 		}
 		crlFix()
@@ -1086,25 +1115,36 @@ func (oAdmin *OvpnAdmin) userRotate(username, newPassword string) string {
 			usersFromIndexTxt := indexTxtParser(fRead(*indexTxtPath))
 			for i := range usersFromIndexTxt {
 				if usersFromIndexTxt[i].DistinguishedName == "/CN="+username {
-					usersFromIndexTxt[i].DistinguishedName = "/CN=REVOKED" + username + "-" + uniqHash
+					usersFromIndexTxt[i].DistinguishedName = "/CN=REVOKED-" + username + "-" + uniqHash
 					break
 				}
 			}
-			fWrite(*indexTxtPath, renderIndexTxt(usersFromIndexTxt))
+			err := fWrite(*indexTxtPath, renderIndexTxt(usersFromIndexTxt))
+			if err != nil {
+				log.Error(err)
+			}
+
 			oAdmin.userCreate(username, newPassword)
 			usersFromIndexTxt = indexTxtParser(fRead(*indexTxtPath))
 			for i := range usersFromIndexTxt {
 				if usersFromIndexTxt[i].DistinguishedName == "/CN="+username {
 					newUserIndex = i
 				}
-				if usersFromIndexTxt[i].DistinguishedName == "/CN=REVOKED"+username+"-"+uniqHash {
+				if usersFromIndexTxt[i].DistinguishedName == "/CN=REVOKED-"+username+"-"+uniqHash {
 					oldUserIndex = i
 				}
 			}
 			usersFromIndexTxt[oldUserIndex], usersFromIndexTxt[newUserIndex] = usersFromIndexTxt[newUserIndex], usersFromIndexTxt[oldUserIndex]
 
-			fWrite(*indexTxtPath, renderIndexTxt(usersFromIndexTxt))
+			if *authByPassword {
-			_ = runBash(fmt.Sprintf("cd %s && easyrsa gen-crl", *easyrsaDirPath))
+				_ = runBash(fmt.Sprintf("openvpn-user change-password --db.path %s --user %s --password %s", *authDatabase, username, newPassword))
+			}
+
+			err = fWrite(*indexTxtPath, renderIndexTxt(usersFromIndexTxt))
+			if err != nil {
+				log.Error(err)
+			}
+			_ = runBash(fmt.Sprintf("cd %s && easyrsa gen-crl 1>/dev/null", *easyrsaDirPath))
 		}
 		crlFix()
 		oAdmin.clients = oAdmin.usersList()
@@ -1121,17 +1161,22 @@ func (oAdmin *OvpnAdmin) userDelete(username string) string {
 				log.Error(err)
 			}
 		} else {
-
 			uniqHash := strings.Replace(uuid.New().String(), "-", "", -1)
 			usersFromIndexTxt := indexTxtParser(fRead(*indexTxtPath))
 			for i := range usersFromIndexTxt {
 				if usersFromIndexTxt[i].DistinguishedName == "/CN="+username {
-					usersFromIndexTxt[i].DistinguishedName = "/CN=DELETED" + username + "-" + uniqHash
+					usersFromIndexTxt[i].DistinguishedName = "/CN=REVOKED-" + username + "-" + uniqHash
 					break
 				}
 			}
-			fWrite(*indexTxtPath, renderIndexTxt(usersFromIndexTxt))
+			if *authByPassword {
-			_ = runBash(fmt.Sprintf("cd %s && easyrsa gen-crl", *easyrsaDirPath))
+				_ = runBash(fmt.Sprintf("openvpn-user delete --force --db.path %s --user %s", *authDatabase, username))
+			}
+			err := fWrite(*indexTxtPath, renderIndexTxt(usersFromIndexTxt))
+			if err != nil {
+				log.Error(err)
+			}
+			_ = runBash(fmt.Sprintf("cd %s && easyrsa gen-crl 1>/dev/null ", *easyrsaDirPath))
 		}
 		crlFix()
 		oAdmin.clients = oAdmin.usersList()
@@ -1317,6 +1362,7 @@ func (oAdmin *OvpnAdmin) mgmtSetTimeFormat() {
 func isUserConnected(username string, connectedUsers []clientStatus) (bool, []string) {
 	var connections []string
 	var connected = false
+
 	for _, connectedUser := range connectedUsers {
 		if connectedUser.CommonName == username {
 			connected = true
@@ -1328,8 +1374,12 @@ func isUserConnected(username string, connectedUsers []clientStatus) (bool, []st
 
 func (oAdmin *OvpnAdmin) downloadCerts() bool {
 	if fExist(certsArchivePath) {
-		fDelete(certsArchivePath)
+		err := fDelete(certsArchivePath)
+		if err != nil {
+			log.Error(err)
+		}
 	}
+
 	err := fDownload(certsArchivePath, *masterHost+downloadCertsApiUrl+"?token="+oAdmin.masterSyncToken, oAdmin.masterHostBasicAuth)
 	if err != nil {
 		log.Error(err)
@@ -1341,7 +1391,10 @@ func (oAdmin *OvpnAdmin) downloadCerts() bool {
 
 func (oAdmin *OvpnAdmin) downloadCcd() bool {
 	if fExist(ccdArchivePath) {
-		fDelete(ccdArchivePath)
+		err := fDelete(ccdArchivePath)
+		if err != nil {
+			log.Error(err)
+		}
 	}
 
 	err := fDownload(ccdArchivePath, *masterHost+downloadCcdApiUrl+"?token="+oAdmin.masterSyncToken, oAdmin.masterHostBasicAuth)
@@ -1354,55 +1407,69 @@ func (oAdmin *OvpnAdmin) downloadCcd() bool {
 }
 
 func archiveCerts() {
-	o := runBash(fmt.Sprintf("cd %s && tar -czf %s *", *easyrsaDirPath+"/pki", certsArchivePath))
+	err := createArchiveFromDir(*easyrsaDirPath+"/pki", certsArchivePath)
-	log.Trace(o)
+	if err != nil {
+		log.Warnf("archiveCerts(): %s", err)
+	}
 }
 
 func archiveCcd() {
-	o := runBash(fmt.Sprintf("cd %s && tar -czf %s *", *ccdDir, ccdArchivePath))
+	err := createArchiveFromDir(*ccdDir, ccdArchivePath)
-	log.Trace(o)
+	if err != nil {
+		log.Warnf("archiveCcd(): %s", err)
+	}
 }
 
 func unArchiveCerts() {
-	runBash(fmt.Sprintf("mkdir -p %s", *easyrsaDirPath+"/pki"))
+	if err := os.MkdirAll(*easyrsaDirPath+"/pki", 0755); err != nil {
-	o := runBash(fmt.Sprintf("cd %s && tar -xzf %s", *easyrsaDirPath+"/pki", certsArchivePath))
+		log.Warnf("unArchiveCerts(): error creating pki dir: %s", err)
-	log.Trace(o)
+	}
+
+	err := extractFromArchive(certsArchivePath, *easyrsaDirPath+"/pki")
+	if err != nil {
+		log.Warnf("unArchiveCerts: extractFromArchive() %s", err)
+	}
 }
 
 func unArchiveCcd() {
-	runBash(fmt.Sprintf("mkdir -p %s", *ccdDir))
+	if err := os.MkdirAll(*ccdDir, 0755); err != nil {
-	o := runBash(fmt.Sprintf("cd %s && tar -xzf %s", *ccdDir, ccdArchivePath))
+		log.Warnf("unArchiveCcd(): error creating ccd dir: %s", err)
-	log.Trace(o)
+	}
+
+	err := extractFromArchive(ccdArchivePath, *ccdDir)
+	if err != nil {
+		log.Warnf("unArchiveCcd: extractFromArchive() %s", err)
+	}
 }
 
 func (oAdmin *OvpnAdmin) syncDataFromMaster() {
 	retryCountMax := 3
 	certsDownloadFailed := true
 	ccdDownloadFailed := true
-	certsDownloadRetries := 0
-	ccdDownloadRetries := 0
 
-	for certsDownloadFailed && certsDownloadRetries < retryCountMax {
+	for certsDownloadRetries := 0; certsDownloadRetries < retryCountMax; certsDownloadRetries++ {
-		certsDownloadRetries += 1
+		log.Infof("Downloading archive with certificates from master. Attempt %d", certsDownloadRetries)
-		log.Infof("Downloading certs archive from master. Attempt %d", certsDownloadRetries)
 		if oAdmin.downloadCerts() {
 			certsDownloadFailed = false
-			log.Info("Decompression certs archive from master")
+			log.Info("Decompressing archive with certificates from master")
 			unArchiveCerts()
+			log.Info("Decompression archive with certificates from master completed")
+			break
 		} else {
-			log.Warnf("something goes wrong during downloading certs from master. Attempt %d", certsDownloadRetries)
+			log.Warnf("Something goes wrong during downloading archive with certificates from master. Attempt %d", certsDownloadRetries)
 		}
 	}
 
-	for ccdDownloadFailed && ccdDownloadRetries < retryCountMax {
+	for ccdDownloadRetries := 0; ccdDownloadRetries < retryCountMax; ccdDownloadRetries++ {
-		ccdDownloadRetries += 1
+		log.Infof("Downloading archive with ccd from master. Attempt %d", ccdDownloadRetries)
-		log.Infof("Downloading ccd archive from master. Attempt %d", ccdDownloadRetries)
 		if oAdmin.downloadCcd() {
 			ccdDownloadFailed = false
-			log.Info("Decompression ccd archive from master")
+			log.Info("Decompressing archive with ccd from master")
 			unArchiveCcd()
+			log.Info("Decompression archive with ccd from master completed")
+			break
 		} else {
-			log.Warnf("something goes wrong during downloading certs from master. Attempt %d", ccdDownloadRetries)
+			log.Warnf("Something goes wrong during downloading archive with ccd from master. Attempt %d", ccdDownloadRetries)
 		}
 	}
 
@@ -1439,9 +1506,9 @@ func getOvpnServerHostsFromKubeApi() ([]OpenvpnServer, error) {
 			log.Error(err)
 		}
 
-		log.Tracef("Debug: service from kube api %v", service)
+		log.Tracef("service from kube api %v", service)
-		log.Tracef("Debug: service.Status from kube api %v", service.Status)
+		log.Tracef("service.Status from kube api %v", service.Status)
-		log.Tracef("Debug: service.Status.LoadBalancer from kube api %v", service.Status.LoadBalancer)
+		log.Tracef("service.Status.LoadBalancer from kube api %v", service.Status.LoadBalancer)
 
 		lbIngress := service.Status.LoadBalancer.Ingress
 		if len(lbIngress) > 0 {