feat: filebrowser with encfs

This commit is contained in:
2024-09-03 00:26:36 +08:00
parent 14024c56a3
commit b0b9f70129
330 changed files with 44745 additions and 87 deletions

370
encfs/file.go Normal file
View File

@@ -0,0 +1,370 @@
package encfs
import (
"crypto/aes"
"crypto/rand"
"encoding/binary"
"encoding/json"
"errors"
"io"
"io/fs"
"math"
"os"
"strings"
"syscall"
"github.com/spf13/afero"
)
const EncFileExt = ".__encfile"
var (
ErrFileForbiddenFileExt = errors.New("file ext is forbidden")
)
type EncFileMeta struct {
Name string `json:"name"`
Iv []byte `json:"iv"`
}
func openOrNewEncFileMeta(name string) (*EncFileMeta, error) {
oldEncFileMeta, err := openEncFileMeta(name)
if err == nil && oldEncFileMeta != nil {
return oldEncFileMeta, nil
}
iv := make([]byte, 16)
_, err = rand.Read(iv)
if err != nil {
return nil, err
}
encFileMeta := &EncFileMeta{
Name: name,
Iv: iv,
}
encFileMetaName := name + EncFileExt
encFileMetaFile, err := os.Create(encFileMetaName)
if err != nil {
return nil, err
}
defer func() {
_ = encFileMetaFile.Close()
}()
encFileMetaBytes, err := marshalEncFileMeta(encFileMeta)
if err != nil {
return nil, err
}
_, err = encFileMetaFile.Write(encFileMetaBytes)
if err != nil {
return nil, err
}
return encFileMeta, nil
}
func openEncFileMeta(name string) (*EncFileMeta, error) {
encFileMetaName := name + EncFileExt
encFileMetaFile, err := os.Open(encFileMetaName)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
defer func() {
_ = encFileMetaFile.Close()
}()
encFileMetaBytes, err := io.ReadAll(encFileMetaFile)
if err != nil {
return nil, err
}
encFileMeta, err := unmarchalEncFileMeta(encFileMetaBytes)
if err != nil {
return nil, err
}
return encFileMeta, nil
}
func marshalEncFileMeta(encFileMeata *EncFileMeta) ([]byte, error) {
return json.Marshal(encFileMeata)
}
func unmarchalEncFileMeta(data []byte) (*EncFileMeta, error) {
var encFileMeta EncFileMeta
err := json.Unmarshal(data, &encFileMeta)
if err != nil {
return nil, err
}
return &encFileMeta, nil
}
type EncFile struct {
isDir bool
closed bool
encFileMeta *EncFileMeta
encFs *EncFs
filePos int64
file *os.File
}
func NewEncFile(name string, file *os.File, encFs *EncFs, isCreate bool) (*EncFile, error) {
fileInfo, err := file.Stat()
if err != nil {
return nil, err
}
isDir := fileInfo.IsDir()
var encFileMeta *EncFileMeta = nil
fileState, err := os.Stat(name)
if err == nil && !fileState.IsDir() && fileState.Size() == 0 {
isCreate = true
}
if !isDir {
if isCreate {
encFileMeta, err = openOrNewEncFileMeta(name)
} else {
encFileMeta, err = openEncFileMeta(name)
}
if err != nil {
return nil, err
}
}
return &EncFile{
isDir: isDir,
closed: false,
encFileMeta: encFileMeta,
encFs: encFs,
filePos: 0,
file: file,
}, nil
}
func (f *EncFile) Close() error {
if f.closed {
return afero.ErrFileClosed
}
f.closed = true
return f.file.Close()
}
func (f *EncFile) Read(p []byte) (n int, err error) {
checkIsFileErr := f.checkIsFile()
if checkIsFileErr != nil {
return 0, checkIsFileErr
}
beforeReadFilePos := f.filePos
readLen, err := f.file.Read(p)
if err == nil {
f.filePos += int64(readLen)
if f.encFs != nil && f.encFs.key != nil && f.encFileMeta != nil {
encryptedBytes, err := generateCtrEncryptBytes(f.encFs.key.key, f.encFileMeta.Iv, beforeReadFilePos, int64(readLen))
if err != nil {
return 0, err
}
for i := 0; i < readLen; i++ {
p[i] = p[i] ^ encryptedBytes[i]
}
}
}
return readLen, err
}
func (f *EncFile) ReadAt(p []byte, off int64) (n int, err error) {
checkIsFileErr := f.checkIsFile()
if checkIsFileErr != nil {
return 0, checkIsFileErr
}
readLen, err := f.file.ReadAt(p, off)
if err == nil {
if f.encFs != nil && f.encFs.key != nil && f.encFileMeta != nil {
encryptedBytes, err := generateCtrEncryptBytes(f.encFs.key.key, f.encFileMeta.Iv, off, int64(readLen))
if err != nil {
return 0, err
}
for i := 0; i < readLen; i++ {
p[i] = p[i] ^ encryptedBytes[i]
}
}
}
return readLen, err
}
func (f *EncFile) Seek(offset int64, whence int) (int64, error) {
checkIsFileErr := f.checkIsFile()
if checkIsFileErr != nil {
return 0, checkIsFileErr
}
ret, err := f.file.Seek(offset, whence)
if err == nil {
f.filePos = ret
}
return ret, err
}
func (f *EncFile) Write(p []byte) (n int, err error) {
checkIsFileErr := f.checkIsFile()
if checkIsFileErr != nil {
return 0, checkIsFileErr
}
writeBuff := p
if f.encFs != nil && f.encFs.key != nil && f.encFileMeta != nil {
buff := make([]byte, len(p))
encryptedBytes, err := generateCtrEncryptBytes(f.encFs.key.key, f.encFileMeta.Iv, f.filePos, int64(len(p)))
if err != nil {
return 0, err
}
for i := 0; i < len(p); i++ {
buff[i] = p[i] ^ encryptedBytes[i]
}
writeBuff = buff
}
writeLen, err := f.file.Write(writeBuff)
if err == nil {
f.filePos += int64(writeLen)
}
return writeLen, err
}
func (f *EncFile) WriteAt(p []byte, off int64) (n int, err error) {
checkIsFileErr := f.checkIsFile()
if checkIsFileErr != nil {
return 0, checkIsFileErr
}
writeBuff := p
if f.encFs != nil && f.encFs.key != nil && f.encFileMeta != nil {
buff := make([]byte, len(p))
encryptedBytes, err := generateCtrEncryptBytes(f.encFs.key.key, f.encFileMeta.Iv, off, int64(len(p)))
if err != nil {
return 0, err
}
for i := 0; i < len(p); i++ {
buff[i] = p[i] ^ encryptedBytes[i]
}
writeBuff = buff
}
writeLen, err := f.file.WriteAt(writeBuff, off)
return writeLen, err
}
func (f *EncFile) Name() string {
return f.file.Name()
}
func (f *EncFile) Readdir(count int) ([]os.FileInfo, error) {
if f.closed {
return nil, afero.ErrFileClosed
}
// FIXME is count * 2 just ok?
fileInfos, err := f.file.Readdir(count * 2)
if err != nil {
return nil, err
}
if count == 0 {
return make([]fs.FileInfo, 0), nil
}
filterFileInfos := make([]os.FileInfo, 0)
for _, fileInfo := range fileInfos {
isEncFileMetaFile := strings.HasSuffix(fileInfo.Name(), EncFileExt)
if !isEncFileMetaFile {
filterFileInfos = append(filterFileInfos, fileInfo)
}
if count >= 0 && len(filterFileInfos) >= count {
break
}
}
return filterFileInfos, nil
}
func (f *EncFile) Readdirnames(n int) ([]string, error) {
fi, err := f.Readdir(n)
if err != nil {
return nil, err
}
var names []string
for _, f := range fi {
names = append(names, f.Name())
}
return names, nil
}
func (f *EncFile) Stat() (os.FileInfo, error) {
return f.file.Stat()
}
func (f *EncFile) Sync() error {
return f.file.Sync()
}
func (f *EncFile) Truncate(size int64) error {
return f.file.Truncate(size)
}
func (f *EncFile) WriteString(s string) (ret int, err error) {
return f.Write([]byte(s))
}
func (f *EncFile) checkIsFile() error {
if f.closed {
return afero.ErrFileClosed
}
if f.isDir {
return syscall.EISDIR
}
return nil
}
func generateCtrEncryptBytes(key, iv []byte, offset, len int64) ([]byte, error) {
endOffset := offset + len
blockOffset := offset / 16
encryptStartOffset := blockOffset * 16
encryptEndOffset := (endOffset / 16) * 16
if endOffset%16 > 0 {
encryptEndOffset += 16
}
blocksCount := int((encryptEndOffset - encryptStartOffset) / 16)
cipher, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
encryptBytes := make([]byte, blocksCount*16)
for i := 0; i < blocksCount; i++ {
encNonce := nonceAdd(iv, uint64(i)+uint64(blockOffset))
cipher.Encrypt(encryptBytes[i*16:(i+1)*16], encNonce)
}
encryptedBytes := encryptBytes[offset-encryptStartOffset : offset-encryptStartOffset+len]
//fmt.Println("XX", hex.EncodeToString(key), hex.EncodeToString(iv), offset, len, hex.EncodeToString(encryptedBytes))
return encryptedBytes, nil
}
func nonceAdd(nonce []byte, incrementValue uint64) []byte {
n1 := binary.BigEndian.Uint64(nonce[:8])
n2 := binary.BigEndian.Uint64(nonce[8:])
leftToMax := math.MaxUint64 - n2
if leftToMax <= incrementValue {
incrementValue -= leftToMax + 1
n2 = incrementValue
n1 += 1
} else {
n2 += incrementValue
}
newNonce := make([]byte, 16)
binary.BigEndian.PutUint64(newNonce, n1)
binary.BigEndian.PutUint64(newNonce[8:], n2)
return newNonce
}

142
encfs/fs.go Normal file
View File

@@ -0,0 +1,142 @@
package encfs
import (
"os"
"strings"
"time"
"github.com/spf13/afero"
)
// copied from afero/os.go
type EncryptionMasterKey struct {
key []byte
}
type EncFs struct {
key *EncryptionMasterKey
}
func NewEncFs(key *EncryptionMasterKey) afero.Fs {
return &EncFs{
key: key,
}
}
func (*EncFs) Name() string { return "EncFs" }
func (encFs *EncFs) Create(name string) (afero.File, error) {
if err := checkFileExt(name); err != nil {
return nil, err
}
f, e := os.Create(name)
if f == nil {
// while this looks strange, we need to return a bare nil (of type nil) not
// a nil value of type *os.File or nil won't be nil
return nil, e
}
return convertOsFileToEncFile(name, f, e, encFs, true)
}
func (*EncFs) Mkdir(name string, perm os.FileMode) error {
return os.Mkdir(name, perm)
}
func (*EncFs) MkdirAll(path string, perm os.FileMode) error {
return os.MkdirAll(path, perm)
}
func (encFs *EncFs) Open(name string) (afero.File, error) {
if err := checkFileExt(name); err != nil {
return nil, err
}
f, e := os.Open(name)
if f == nil {
// while this looks strange, we need to return a bare nil (of type nil) not
// a nil value of type *os.File or nil won't be nil
return nil, e
}
return convertOsFileToEncFile(name, f, e, encFs, false)
}
func (encFs *EncFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
if err := checkFileExt(name); err != nil {
return nil, err
}
f, e := os.OpenFile(name, flag, perm)
if f == nil {
// while this looks strange, we need to return a bare nil (of type nil) not
// a nil value of type *os.File or nil won't be nil
return nil, e
}
return convertOsFileToEncFile(name, f, e, encFs, false)
}
func (*EncFs) Remove(name string) error {
encFileMetaName := name + EncFileExt
_ = os.Remove(encFileMetaName)
return os.Remove(name)
}
func (encFs *EncFs) RemoveAll(path string) error {
fileInfo, err := os.Stat(path)
if err == nil && !fileInfo.IsDir() {
return encFs.Remove(path)
}
return os.RemoveAll(path)
}
func (*EncFs) Rename(oldname, newname string) error {
oldEncFileMetaName := oldname + EncFileExt
newEncFileMetaName := newname + EncFileExt
_ = os.Rename(oldEncFileMetaName, newEncFileMetaName)
return os.Rename(oldname, newname)
}
func (*EncFs) Stat(name string) (os.FileInfo, error) {
return os.Stat(name)
}
func (*EncFs) Chmod(name string, mode os.FileMode) error {
return os.Chmod(name, mode)
}
func (*EncFs) Chown(name string, uid, gid int) error {
return os.Chown(name, uid, gid)
}
func (*EncFs) Chtimes(name string, atime time.Time, mtime time.Time) error {
return os.Chtimes(name, atime, mtime)
}
func (*EncFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
fi, err := os.Lstat(name)
return fi, true, err
}
func (*EncFs) SymlinkIfPossible(oldname, newname string) error {
return os.Symlink(oldname, newname)
}
func (*EncFs) ReadlinkIfPossible(name string) (string, error) {
return os.Readlink(name)
}
func checkFileExt(name string) error {
if strings.HasSuffix(name, EncFileExt) {
return ErrFileForbiddenFileExt
}
return nil
}
func convertOsFileToEncFile(name string, file *os.File, e error, encFs *EncFs, isCreate bool) (afero.File, error) {
if e != nil {
return nil, e
}
encFile, err := NewEncFile(name, file, encFs, isCreate)
if err != nil {
return nil, err
}
return encFile, nil
}

129
encfs/kms.go Normal file
View File

@@ -0,0 +1,129 @@
package encfs
import (
"bytes"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"strings"
"sync"
"time"
)
const LOCAL_MINI_KMS_ADDRESS = "LOCAL_MINI_KMS_ADDRESS"
const DISABLE_ENCRYPTION_MASTER_KEY = "DISABLE_ENCRYPTION_MASTER_KEY"
const ENCRYPTED_ENCRYPTION_MASTER_KEY = "ENCRYPTED_ENCRYPTION_MASTER_KEY"
type MultiViewValue struct {
ValueHex string `json:"value_hex"`
ValueBase64 string `json:"value_base64"`
}
type EncryptRequest struct {
EncryptedValue string `json:"encrypted_value"`
}
var cachedcEncryptionMasterKey *EncryptionMasterKey = nil
var cachedcEncryptionMasterKeyLock sync.Mutex
func GetCachedEncryptionMasterKey() (*EncryptionMasterKey, error) {
cachedcEncryptionMasterKeyLock.Lock()
defer cachedcEncryptionMasterKeyLock.Unlock()
if cachedcEncryptionMasterKey == nil {
var err error
cachedcEncryptionMasterKey, err = GetEncryptionMasterKey()
if err != nil {
return nil, err
}
}
return cachedcEncryptionMasterKey, nil
}
func GetEncryptionMasterKey() (*EncryptionMasterKey, error) {
disableEncryptionMasterKey := os.Getenv(DISABLE_ENCRYPTION_MASTER_KEY)
if disableEncryptionMasterKey == "1" ||
disableEncryptionMasterKey == "on" ||
disableEncryptionMasterKey == "yes" ||
disableEncryptionMasterKey == "true" {
return nil, nil
}
encryptedEncryptionMasterKey := os.Getenv(ENCRYPTED_ENCRYPTION_MASTER_KEY)
if encryptedEncryptionMasterKey == "" {
fmt.Println("[ERROR] encrypted encryption master key is not present")
return nil, errors.New("encrypted encryption master key is not present")
}
key, err := DecryptBytes(encryptedEncryptionMasterKey)
if err != nil {
return nil, err
}
return &EncryptionMasterKey{
key: key,
}, nil
}
func DecryptBytes(encryptedValue string) ([]byte, error) {
localMiniKmsAddress := os.Getenv(LOCAL_MINI_KMS_ADDRESS)
if localMiniKmsAddress == "" {
localMiniKmsAddress = "127.0.0.1:5567"
}
if !strings.HasPrefix(strings.ToLower(localMiniKmsAddress), "http://") {
localMiniKmsAddress = fmt.Sprintf("http://%s", localMiniKmsAddress)
}
multiViewValue, err := Decrypt(localMiniKmsAddress, encryptedValue)
if err != nil {
fmt.Println("[ERROR] Decrypt from", localMiniKmsAddress, "failed, error:", err)
return nil, err
}
valueBytes, err := hex.DecodeString(multiViewValue.ValueHex)
if err != nil {
return nil, err
}
return valueBytes, nil
}
func Decrypt(endpoint, encryptedValue string) (*MultiViewValue, error) {
encryptRequest := EncryptRequest{
EncryptedValue: encryptedValue,
}
encryptRequestBytes, err := json.Marshal(&encryptRequest)
if err != nil {
return nil, err
}
encryptRequestReader := bytes.NewReader(encryptRequestBytes)
client := http.Client{
Timeout: 5 * time.Second,
}
encryptResponse, err := client.Post(joinEnpointPath(endpoint, "/decrypt"), "application/json", encryptRequestReader)
if err != nil {
return nil, err
}
if encryptResponse.StatusCode != 200 {
return nil, fmt.Errorf("decrypt failed, http status: %d", encryptResponse.StatusCode)
}
encryptResponseBodyBytes, err := io.ReadAll(encryptResponse.Body)
if err != nil {
return nil, err
}
var multiViewValue MultiViewValue
err = json.Unmarshal(encryptResponseBodyBytes, &multiViewValue)
if err != nil {
return nil, err
}
return &multiViewValue, nil
}
func joinEnpointPath(endpoint, path string) string {
endpointEndsWithSlash := strings.HasSuffix(endpoint, "/")
pathStartsWithSlash := strings.HasPrefix(path, "/")
if endpointEndsWithSlash && pathStartsWithSlash {
return fmt.Sprintf("%s%s", endpoint, path[1:])
} else if endpointEndsWithSlash || pathStartsWithSlash {
return fmt.Sprintf("%s%s", endpoint, path)
} else {
return fmt.Sprintf("%s/%s", endpoint, path)
}
}