feat: 文件格式导入支持扩展 - 后端

go.mod

@@ -14,9 +14,14 @@ require (
 	github.com/alibabacloud-go/rds-20140815/v2 v2.1.0
 	github.com/alibabacloud-go/tea v1.1.19
 	github.com/alibabacloud-go/tea-utils v1.4.3
+	github.com/andybalholm/brotli v1.0.4 // indirect
 	github.com/baidubce/bce-sdk-go v0.9.151
+	github.com/bodgit/plumbing v1.2.0 // indirect
+	github.com/bodgit/sevenzip v1.3.0
+	github.com/bodgit/windows v1.0.0 // indirect
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/clbanning/mxj/v2 v2.5.6 // indirect
+	github.com/connesc/cipherio v0.2.1 // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548
 	github.com/cznic/parser v0.0.0-20181122101858-d773202d5b1f
 	github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8
@@ -36,15 +41,20 @@ require (
 	github.com/go-sql-driver/mysql v1.7.0
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang/protobuf v1.5.3
+	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-hclog v0.14.1
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-plugin v1.4.2
 	github.com/jackc/pgx/v4 v4.13.0
 	github.com/jmoiron/sqlx v1.3.3
+	github.com/klauspost/compress v1.15.9 // indirect
 	github.com/labstack/echo/v4 v4.10.2
 	github.com/larksuite/oapi-sdk-go/v3 v3.0.23
 	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
+	github.com/nwaples/rardecode v1.1.3
 	github.com/openark/golib v0.0.0-20210531070646-355f37940af8
+	github.com/pierrec/lz4/v4 v4.1.15 // indirect
 	github.com/pingcap/parser v3.0.12+incompatible
 	github.com/pingcap/tidb v1.1.0-beta.0.20200630082100-328b6d0a955c
 	github.com/pkg/errors v0.9.1
@@ -54,6 +64,7 @@ require (
 	github.com/spf13/cobra v1.1.1
 	github.com/stretchr/testify v1.8.4
 	github.com/swaggo/swag v1.16.3
+	github.com/ulikunitz/xz v0.5.10 // indirect
 	github.com/ungerik/go-dry v0.0.0-20210209114055-a3e162a9e62e
 	github.com/urfave/cli/v2 v2.8.1
 	golang.org/x/net v0.17.0
@@ -64,10 +75,12 @@ require (
 )
 
 require (
+	github.com/360EntSecGroup-Skylar/excelize v1.4.1
 	github.com/aliyun/credentials-go v1.1.2
 	github.com/hashicorp/go-version v1.7.0
 	github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.1.69
 	github.com/nicksnyder/go-i18n/v2 v2.4.0
+	github.com/xuri/excelize/v2 v2.7.1
 	golang.org/x/crypto v0.14.0
 	golang.org/x/text v0.14.0
 	gorm.io/driver/mysql v1.4.7
@@ -76,7 +89,6 @@ require (
 
 require (
 	dario.cat/mergo v1.0.0 // indirect
-	github.com/360EntSecGroup-Skylar/excelize v1.4.1 // indirect
 	github.com/BurntSushi/toml v1.3.2 // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
@@ -151,6 +163,8 @@ require (
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
+	github.com/richardlehane/mscfb v1.0.4 // indirect
+	github.com/richardlehane/msoleps v1.0.3 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/sergi/go-diff v1.1.0 // indirect
@@ -165,10 +179,13 @@ require (
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 // indirect
+	github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22 // indirect
 	go.mongodb.org/mongo-driver v1.12.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.17.0 // indirect
+	go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
 	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect

sqle/api/controller/v1/archive_7z.go

@@ -0,0 +1,149 @@
+package v1
+
+import (
+	"bytes"
+	e "errors"
+	"fmt"
+	"io"
+	"net/http"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	xmlParser "github.com/actiontech/mybatis-mapper-2-sql"
+	"github.com/actiontech/sqle/sqle/log"
+	"github.com/actiontech/sqle/sqle/utils"
+	"github.com/bodgit/sevenzip"
+	"github.com/labstack/echo/v4"
+)
+
+// getSqlsFrom7z 从 7z 文件中提取 SQL 语句。
+// 使用 github.com/bodgit/sevenzip 库解压 7z 文件，遍历内部文件并调用 processArchiveEntry 处理。
+// 函数签名与 getSqlsFromZip / getSqlsFromRar 保持一致。
+// 注意：sevenzip 需要 io.ReaderAt + int64 size（不同于 RAR 的 io.Reader），需先将上传文件读入 bytes.Reader。
+func getSqlsFrom7z(c echo.Context) (sqlsFromSQLFile []SQLsFromSQLFile, sqlsFromXML []SQLFromXML, skippedCount int, exist bool, err error) {
+	file, err := c.FormFile(InputZipFileName)
+	if err == http.ErrMissingFile {
+		return nil, nil, 0, false, nil
+	}
+	if err != nil {
+		return nil, nil, 0, false, err
+	}
+
+	f, err := file.Open()
+	if err != nil {
+		return nil, nil, 0, false, err
+	}
+	defer f.Close()
+
+	// 使用 archiveConfig 进行压缩包总大小限制检查（上传文件大小预检）
+	if err := defaultArchiveConfig.checkSize(0, file.Size); err != nil {
+		return nil, nil, 0, false, err
+	}
+
+	// sevenzip 需要 io.ReaderAt 接口，将上传文件内容读入 bytes.Reader
+	buf, err := io.ReadAll(f)
+	if err != nil {
+		return nil, nil, 0, false, fmt.Errorf("read 7z file into memory failed: %v", err)
+	}
+
+	sqlsFromSQLFile, sqlsFromXML, skippedCount, err = process7zContent(bytes.NewReader(buf), int64(len(buf)))
+	if err != nil {
+		return nil, nil, 0, false, err
+	}
+
+	return sqlsFromSQLFile, sqlsFromXML, skippedCount, true, nil
+}
+
+// process7zContent 从 io.ReaderAt 中读取 7z 内容，遍历 entry 并提取 SQL。
+// 该函数封装了 7z 解压的核心逻辑，独立于 echo.Context，便于单元测试。
+func process7zContent(r io.ReaderAt, size int64) (sqlsFromSQLFile []SQLsFromSQLFile, sqlsFromXML []SQLFromXML, skippedCount int, err error) {
+	// 使用 sevenzip.NewReader 打开 7z 文件
+	szr, err := sevenzip.NewReader(r, size)
+	if err != nil {
+		return nil, nil, 0, fmt.Errorf("open 7z file failed: %v", err)
+	}
+
+	var xmlContents []xmlParser.XmlFile
+	var totalSize int64
+	var fileCount int
+
+	for _, f := range szr.File {
+		// 跳过目录
+		if f.FileInfo().IsDir() {
+			continue
+		}
+
+		// 文件数量限制检查
+		fileCount++
+		if err := defaultArchiveConfig.checkFileCount(fileCount); err != nil {
+			return nil, nil, 0, err
+		}
+
+		// 嵌套压缩包检查：depth=1 时跳过内层压缩包
+		ext := strings.ToLower(filepath.Ext(f.Name))
+		if supportedArchiveExts[ext] {
+			continue
+		}
+
+		// 打开并读取文件内容
+		rc, err := f.Open()
+		if err != nil {
+			return nil, nil, 0, fmt.Errorf("open 7z entry %q failed: %v", f.Name, err)
+		}
+		content, err := io.ReadAll(rc)
+		rc.Close()
+		if err != nil {
+			return nil, nil, 0, fmt.Errorf("read 7z entry %q content failed: %v", f.Name, err)
+		}
+
+		// 累计大小限制检查
+		totalSize += int64(len(content))
+		if err := defaultArchiveConfig.checkSize(0, totalSize); err != nil {
+			return nil, nil, 0, err
+		}
+
+		// 委托 processArchiveEntry 按扩展名分发处理
+		sqlContent, xmlContent, isSupported, err := processArchiveEntry(f.Name, content)
+		if err != nil {
+			if e.Is(err, utils.ErrUnknownEncoding) {
+				log.NewEntry().WithField("convert_to_utf8", f.Name).Errorf("convert to utf8 failed: %v", err)
+				continue
+			}
+			return nil, nil, 0, err
+		}
+		if !isSupported {
+			skippedCount++
+			continue
+		}
+
+		if xmlContent != nil {
+			xmlContents = append(xmlContents, *xmlContent)
+		} else if sqlContent != "" {
+			sqlsFromSQLFile = append(sqlsFromSQLFile, SQLsFromSQLFile{
+				FilePath: f.Name,
+				SQLs:     sqlContent,
+			})
+		}
+	}
+
+	// parse xml content
+	// xml文件需要把所有文件内容同时解析，否则会无法解析跨namespace引用的SQL
+	{
+		sqlsFromXmls, err := parseXMLsWithFilePath(xmlContents)
+		if err != nil {
+			return nil, nil, 0, err
+		}
+		sqlsFromXML = append(sqlsFromXML, sqlsFromXmls...)
+	}
+
+	// 按文件名自然排序，确保SQL按文件顺序执行
+	sort.Slice(sqlsFromSQLFile, func(i, j int) bool {
+		return utils.CompareNatural(sqlsFromSQLFile[i].FilePath, sqlsFromSQLFile[j].FilePath)
+	})
+	sort.Slice(sqlsFromXML, func(i, j int) bool {
+		return utils.CompareNatural(sqlsFromXML[i].FilePath, sqlsFromXML[j].FilePath)
+	})
+
+	return sqlsFromSQLFile, sqlsFromXML, skippedCount, nil
+}