This commit is contained in:
parent
5dedae3f97
commit
55d7dee111
@ -36,3 +36,7 @@ type ImportProvinceResponse struct {
|
|||||||
IsAsync bool `json:"is_async"`
|
IsAsync bool `json:"is_async"`
|
||||||
Message string `json:"message,omitempty"`
|
Message string `json:"message,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type StopImportRequest struct {
|
||||||
|
JobID string `json:"job_id" binding:"required"`
|
||||||
|
}
|
||||||
|
|||||||
@ -36,6 +36,7 @@ func (h *CityHandler) Router() {
|
|||||||
r.DELETE("/:id", h.Delete)
|
r.DELETE("/:id", h.Delete)
|
||||||
r.GET("/template", h.Template)
|
r.GET("/template", h.Template)
|
||||||
r.POST("/import", h.Import)
|
r.POST("/import", h.Import)
|
||||||
|
r.POST("/import/stop", h.ImportStop)
|
||||||
r.GET("/import/status/:jobId", h.ImportStatus)
|
r.GET("/import/status/:jobId", h.ImportStatus)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -254,3 +255,27 @@ func (h *CityHandler) ImportStatus(c *gin.Context) {
|
|||||||
"logs": logs,
|
"logs": logs,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *CityHandler) ImportStop(c *gin.Context) {
|
||||||
|
userid, err := h.Helper.GetAuthInfo(c.GetString("auth"))
|
||||||
|
if err != nil {
|
||||||
|
response.Error(c, http.StatusUnauthorized, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var req dto.StopImportRequest
|
||||||
|
if err := c.BindJSON(&req); err != nil {
|
||||||
|
response.Error(c, http.StatusBadRequest, "job_id is required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
handler := cityimporter.NewCityImportHandler(*userid, h.Helper)
|
||||||
|
imp := importer.NewImporter(h.Helper, handler)
|
||||||
|
|
||||||
|
if err := imp.StopJob(req.JobID); err != nil {
|
||||||
|
response.Error(c, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
response.Success(c, map[string]interface{}{"message": "Job stopped"})
|
||||||
|
}
|
||||||
|
|||||||
@ -36,6 +36,7 @@ func (h *ProvinceHandler) Router() {
|
|||||||
r.DELETE("/:id", h.Delete)
|
r.DELETE("/:id", h.Delete)
|
||||||
r.GET("/template", h.Template)
|
r.GET("/template", h.Template)
|
||||||
r.POST("/import", h.Import)
|
r.POST("/import", h.Import)
|
||||||
|
r.POST("/import/stop", h.ImportStop)
|
||||||
r.GET("/import/status/:jobId", h.ImportStatus)
|
r.GET("/import/status/:jobId", h.ImportStatus)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -254,3 +255,27 @@ func (h *ProvinceHandler) ImportStatus(c *gin.Context) {
|
|||||||
"logs": logs,
|
"logs": logs,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *ProvinceHandler) ImportStop(c *gin.Context) {
|
||||||
|
userid, err := h.Helper.GetAuthInfo(c.GetString("auth"))
|
||||||
|
if err != nil {
|
||||||
|
response.Error(c, http.StatusUnauthorized, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var req dto.StopImportRequest
|
||||||
|
if err := c.BindJSON(&req); err != nil {
|
||||||
|
response.Error(c, http.StatusBadRequest, "job_id is required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
handler := provinceimporter.NewProvinceImportHandler(*userid)
|
||||||
|
imp := importer.NewImporter(h.Helper, handler)
|
||||||
|
|
||||||
|
if err := imp.StopJob(req.JobID); err != nil {
|
||||||
|
response.Error(c, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
response.Success(c, map[string]interface{}{"message": "Job stopped"})
|
||||||
|
}
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
package csv
|
package csv
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"encoding/csv"
|
"encoding/csv"
|
||||||
"io"
|
"io"
|
||||||
"strings"
|
"strings"
|
||||||
@ -9,6 +10,10 @@ import (
|
|||||||
type RowHandler func(row []string, rowNumber int) error
|
type RowHandler func(row []string, rowNumber int) error
|
||||||
|
|
||||||
func ReadCSV(file io.Reader, handler RowHandler) (int, []string, error) {
|
func ReadCSV(file io.Reader, handler RowHandler) (int, []string, error) {
|
||||||
|
return ReadCSVWithContext(context.Background(), file, handler)
|
||||||
|
}
|
||||||
|
|
||||||
|
func ReadCSVWithContext(ctx context.Context, file io.Reader, handler RowHandler) (int, []string, error) {
|
||||||
reader := csv.NewReader(file)
|
reader := csv.NewReader(file)
|
||||||
reader.LazyQuotes = true
|
reader.LazyQuotes = true
|
||||||
reader.TrimLeadingSpace = true
|
reader.TrimLeadingSpace = true
|
||||||
@ -18,6 +23,12 @@ func ReadCSV(file io.Reader, handler RowHandler) (int, []string, error) {
|
|||||||
totalRows := 0
|
totalRows := 0
|
||||||
|
|
||||||
for {
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return totalRows, errors, ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
row, err := reader.Read()
|
row, err := reader.Read()
|
||||||
if err == io.EOF {
|
if err == io.EOF {
|
||||||
break
|
break
|
||||||
@ -29,17 +40,18 @@ func ReadCSV(file io.Reader, handler RowHandler) (int, []string, error) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Skip empty rows
|
|
||||||
if len(row) == 0 || (len(row) == 1 && strings.TrimSpace(row[0]) == "") {
|
if len(row) == 0 || (len(row) == 1 && strings.TrimSpace(row[0]) == "") {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Skip header row (first row)
|
|
||||||
if rowNumber == 1 {
|
if rowNumber == 1 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := handler(row, rowNumber); err != nil {
|
if err := handler(row, rowNumber); err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return totalRows, errors, ctx.Err()
|
||||||
|
}
|
||||||
errors = append(errors, "Row "+itoa(rowNumber)+": "+err.Error())
|
errors = append(errors, "Row "+itoa(rowNumber)+": "+err.Error())
|
||||||
}
|
}
|
||||||
totalRows++
|
totalRows++
|
||||||
|
|||||||
@ -12,12 +12,18 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
cancelFuncs = make(map[string]context.CancelFunc)
|
||||||
|
cancelMu sync.RWMutex
|
||||||
|
)
|
||||||
|
|
||||||
type RowData interface {
|
type RowData interface {
|
||||||
TableName() string
|
TableName() string
|
||||||
}
|
}
|
||||||
@ -231,7 +237,11 @@ func (i *Importer) processSync(file io.Reader, userID string, totalRows int) (*I
|
|||||||
|
|
||||||
func (i *Importer) processAsync(file io.Reader, userID string, totalRows int) (*ImportResult, error) {
|
func (i *Importer) processAsync(file io.Reader, userID string, totalRows int) (*ImportResult, error) {
|
||||||
jobID := uuid.New().String()
|
jobID := uuid.New().String()
|
||||||
ctx := context.Background()
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|
||||||
|
cancelMu.Lock()
|
||||||
|
cancelFuncs[jobID] = cancel
|
||||||
|
cancelMu.Unlock()
|
||||||
|
|
||||||
metadata := map[string]string{
|
metadata := map[string]string{
|
||||||
"user_id": userID,
|
"user_id": userID,
|
||||||
@ -239,10 +249,14 @@ func (i *Importer) processAsync(file io.Reader, userID string, totalRows int) (*
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := i.JobQ.CreateJob(ctx, jobID, totalRows, metadata); err != nil {
|
if err := i.JobQ.CreateJob(ctx, jobID, totalRows, metadata); err != nil {
|
||||||
|
cancel()
|
||||||
|
cancelMu.Lock()
|
||||||
|
delete(cancelFuncs, jobID)
|
||||||
|
cancelMu.Unlock()
|
||||||
return nil, fmt.Errorf("failed to create job: %w", err)
|
return nil, fmt.Errorf("failed to create job: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
go i.processAsyncWorker(file, jobID, userID, totalRows)
|
go i.processAsyncWorker(ctx, file, jobID, userID, totalRows)
|
||||||
|
|
||||||
return &ImportResult{
|
return &ImportResult{
|
||||||
JobID: jobID,
|
JobID: jobID,
|
||||||
@ -252,9 +266,8 @@ func (i *Importer) processAsync(file io.Reader, userID string, totalRows int) (*
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (i *Importer) processAsyncWorker(file io.Reader, jobID string, userID string, totalRows int) {
|
func (i *Importer) processAsyncWorker(ctx context.Context, file io.Reader, jobID string, userID string, totalRows int) {
|
||||||
db := i.Helper.GetDB("master")
|
db := i.Helper.GetDB("master")
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
var imported, skipped int
|
var imported, skipped int
|
||||||
var logs []ImportLog
|
var logs []ImportLog
|
||||||
@ -263,9 +276,15 @@ func (i *Importer) processAsyncWorker(file io.Reader, jobID string, userID strin
|
|||||||
var batchRows []int
|
var batchRows []int
|
||||||
batchSize := 500
|
batchSize := 500
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
cancelMu.Lock()
|
||||||
|
delete(cancelFuncs, jobID)
|
||||||
|
cancelMu.Unlock()
|
||||||
|
}()
|
||||||
|
|
||||||
i.JobQ.UpdateProgress(ctx, jobID, 0, 0, nil)
|
i.JobQ.UpdateProgress(ctx, jobID, 0, 0, nil)
|
||||||
|
|
||||||
csvlib.ReadCSV(file, func(row []string, rowNumber int) error {
|
csvlib.ReadCSVWithContext(ctx, file, func(row []string, rowNumber int) error {
|
||||||
data, rowErrors := i.Handler.ValidateRow(row, rowNumber)
|
data, rowErrors := i.Handler.ValidateRow(row, rowNumber)
|
||||||
if len(rowErrors) > 0 {
|
if len(rowErrors) > 0 {
|
||||||
for _, e := range rowErrors {
|
for _, e := range rowErrors {
|
||||||
@ -350,10 +369,27 @@ func (i *Importer) processAsyncWorker(file io.Reader, jobID string, userID strin
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
i.JobQ.CompleteJob(ctx, jobID, imported, skipped, errors)
|
if ctx.Err() != nil {
|
||||||
|
i.JobQ.StopJob(ctx, jobID)
|
||||||
|
} else {
|
||||||
|
i.JobQ.CompleteJob(ctx, jobID, imported, skipped, errors)
|
||||||
|
}
|
||||||
i.saveJobLogs(ctx, jobID, logs, imported, skipped, errors)
|
i.saveJobLogs(ctx, jobID, logs, imported, skipped, errors)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (i *Importer) StopJob(jobID string) error {
|
||||||
|
cancelMu.RLock()
|
||||||
|
cancel, exists := cancelFuncs[jobID]
|
||||||
|
cancelMu.RUnlock()
|
||||||
|
|
||||||
|
if !exists {
|
||||||
|
return fmt.Errorf("job not found or already completed")
|
||||||
|
}
|
||||||
|
|
||||||
|
cancel()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (i *Importer) saveJobLogs(ctx context.Context, jobID string, logs []ImportLog, imported int, skipped int, errors []string) {
|
func (i *Importer) saveJobLogs(ctx context.Context, jobID string, logs []ImportLog, imported int, skipped int, errors []string) {
|
||||||
logsJSON, _ := json.Marshal(logs)
|
logsJSON, _ := json.Marshal(logs)
|
||||||
key := fmt.Sprintf("import_job:%s:logs", jobID)
|
key := fmt.Sprintf("import_job:%s:logs", jobID)
|
||||||
|
|||||||
@ -15,6 +15,7 @@ const (
|
|||||||
StatusProcessing JobStatus = "processing"
|
StatusProcessing JobStatus = "processing"
|
||||||
StatusCompleted JobStatus = "completed"
|
StatusCompleted JobStatus = "completed"
|
||||||
StatusFailed JobStatus = "failed"
|
StatusFailed JobStatus = "failed"
|
||||||
|
StatusStopped JobStatus = "stopped"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Job struct {
|
type Job struct {
|
||||||
@ -124,6 +125,16 @@ func (jq *JobQueue) FailJob(ctx context.Context, jobID string, errMsg string) er
|
|||||||
}).Err()
|
}).Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (jq *JobQueue) StopJob(ctx context.Context, jobID string) error {
|
||||||
|
key := fmt.Sprintf("import_job:%s", jobID)
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
return jq.Redis.HSet(ctx, key, map[string]interface{}{
|
||||||
|
"status": string(StatusStopped),
|
||||||
|
"finished_at": now.Format(time.RFC3339),
|
||||||
|
}).Err()
|
||||||
|
}
|
||||||
|
|
||||||
func (jq *JobQueue) GetJob(ctx context.Context, jobID string) (*Job, error) {
|
func (jq *JobQueue) GetJob(ctx context.Context, jobID string) (*Job, error) {
|
||||||
key := fmt.Sprintf("import_job:%s", jobID)
|
key := fmt.Sprintf("import_job:%s", jobID)
|
||||||
data, err := jq.Redis.HGetAll(ctx, key).Result()
|
data, err := jq.Redis.HGetAll(ctx, key).Result()
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user