stopimport job
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful

This commit is contained in:
teguh nugroho 2026-08-18 14:43:48 +07:00
parent 5dedae3f97
commit 55d7dee111
6 changed files with 121 additions and 8 deletions

View File

@ -36,3 +36,7 @@ type ImportProvinceResponse struct {
IsAsync bool `json:"is_async"` IsAsync bool `json:"is_async"`
Message string `json:"message,omitempty"` Message string `json:"message,omitempty"`
} }
type StopImportRequest struct {
JobID string `json:"job_id" binding:"required"`
}

View File

@ -36,6 +36,7 @@ func (h *CityHandler) Router() {
r.DELETE("/:id", h.Delete) r.DELETE("/:id", h.Delete)
r.GET("/template", h.Template) r.GET("/template", h.Template)
r.POST("/import", h.Import) r.POST("/import", h.Import)
r.POST("/import/stop", h.ImportStop)
r.GET("/import/status/:jobId", h.ImportStatus) r.GET("/import/status/:jobId", h.ImportStatus)
} }
} }
@ -254,3 +255,27 @@ func (h *CityHandler) ImportStatus(c *gin.Context) {
"logs": logs, "logs": logs,
}) })
} }
func (h *CityHandler) ImportStop(c *gin.Context) {
userid, err := h.Helper.GetAuthInfo(c.GetString("auth"))
if err != nil {
response.Error(c, http.StatusUnauthorized, err.Error())
return
}
var req dto.StopImportRequest
if err := c.BindJSON(&req); err != nil {
response.Error(c, http.StatusBadRequest, "job_id is required")
return
}
handler := cityimporter.NewCityImportHandler(*userid, h.Helper)
imp := importer.NewImporter(h.Helper, handler)
if err := imp.StopJob(req.JobID); err != nil {
response.Error(c, http.StatusBadRequest, err.Error())
return
}
response.Success(c, map[string]interface{}{"message": "Job stopped"})
}

View File

@ -36,6 +36,7 @@ func (h *ProvinceHandler) Router() {
r.DELETE("/:id", h.Delete) r.DELETE("/:id", h.Delete)
r.GET("/template", h.Template) r.GET("/template", h.Template)
r.POST("/import", h.Import) r.POST("/import", h.Import)
r.POST("/import/stop", h.ImportStop)
r.GET("/import/status/:jobId", h.ImportStatus) r.GET("/import/status/:jobId", h.ImportStatus)
} }
} }
@ -254,3 +255,27 @@ func (h *ProvinceHandler) ImportStatus(c *gin.Context) {
"logs": logs, "logs": logs,
}) })
} }
func (h *ProvinceHandler) ImportStop(c *gin.Context) {
userid, err := h.Helper.GetAuthInfo(c.GetString("auth"))
if err != nil {
response.Error(c, http.StatusUnauthorized, err.Error())
return
}
var req dto.StopImportRequest
if err := c.BindJSON(&req); err != nil {
response.Error(c, http.StatusBadRequest, "job_id is required")
return
}
handler := provinceimporter.NewProvinceImportHandler(*userid)
imp := importer.NewImporter(h.Helper, handler)
if err := imp.StopJob(req.JobID); err != nil {
response.Error(c, http.StatusBadRequest, err.Error())
return
}
response.Success(c, map[string]interface{}{"message": "Job stopped"})
}

View File

@ -1,6 +1,7 @@
package csv package csv
import ( import (
"context"
"encoding/csv" "encoding/csv"
"io" "io"
"strings" "strings"
@ -9,6 +10,10 @@ import (
type RowHandler func(row []string, rowNumber int) error type RowHandler func(row []string, rowNumber int) error
func ReadCSV(file io.Reader, handler RowHandler) (int, []string, error) { func ReadCSV(file io.Reader, handler RowHandler) (int, []string, error) {
return ReadCSVWithContext(context.Background(), file, handler)
}
func ReadCSVWithContext(ctx context.Context, file io.Reader, handler RowHandler) (int, []string, error) {
reader := csv.NewReader(file) reader := csv.NewReader(file)
reader.LazyQuotes = true reader.LazyQuotes = true
reader.TrimLeadingSpace = true reader.TrimLeadingSpace = true
@ -18,6 +23,12 @@ func ReadCSV(file io.Reader, handler RowHandler) (int, []string, error) {
totalRows := 0 totalRows := 0
for { for {
select {
case <-ctx.Done():
return totalRows, errors, ctx.Err()
default:
}
row, err := reader.Read() row, err := reader.Read()
if err == io.EOF { if err == io.EOF {
break break
@ -29,17 +40,18 @@ func ReadCSV(file io.Reader, handler RowHandler) (int, []string, error) {
continue continue
} }
// Skip empty rows
if len(row) == 0 || (len(row) == 1 && strings.TrimSpace(row[0]) == "") { if len(row) == 0 || (len(row) == 1 && strings.TrimSpace(row[0]) == "") {
continue continue
} }
// Skip header row (first row)
if rowNumber == 1 { if rowNumber == 1 {
continue continue
} }
if err := handler(row, rowNumber); err != nil { if err := handler(row, rowNumber); err != nil {
if ctx.Err() != nil {
return totalRows, errors, ctx.Err()
}
errors = append(errors, "Row "+itoa(rowNumber)+": "+err.Error()) errors = append(errors, "Row "+itoa(rowNumber)+": "+err.Error())
} }
totalRows++ totalRows++

View File

@ -12,12 +12,18 @@ import (
"io" "io"
"net/http" "net/http"
"strings" "strings"
"sync"
"time" "time"
"github.com/google/uuid" "github.com/google/uuid"
"gorm.io/gorm" "gorm.io/gorm"
) )
var (
cancelFuncs = make(map[string]context.CancelFunc)
cancelMu sync.RWMutex
)
type RowData interface { type RowData interface {
TableName() string TableName() string
} }
@ -231,7 +237,11 @@ func (i *Importer) processSync(file io.Reader, userID string, totalRows int) (*I
func (i *Importer) processAsync(file io.Reader, userID string, totalRows int) (*ImportResult, error) { func (i *Importer) processAsync(file io.Reader, userID string, totalRows int) (*ImportResult, error) {
jobID := uuid.New().String() jobID := uuid.New().String()
ctx := context.Background() ctx, cancel := context.WithCancel(context.Background())
cancelMu.Lock()
cancelFuncs[jobID] = cancel
cancelMu.Unlock()
metadata := map[string]string{ metadata := map[string]string{
"user_id": userID, "user_id": userID,
@ -239,10 +249,14 @@ func (i *Importer) processAsync(file io.Reader, userID string, totalRows int) (*
} }
if err := i.JobQ.CreateJob(ctx, jobID, totalRows, metadata); err != nil { if err := i.JobQ.CreateJob(ctx, jobID, totalRows, metadata); err != nil {
cancel()
cancelMu.Lock()
delete(cancelFuncs, jobID)
cancelMu.Unlock()
return nil, fmt.Errorf("failed to create job: %w", err) return nil, fmt.Errorf("failed to create job: %w", err)
} }
go i.processAsyncWorker(file, jobID, userID, totalRows) go i.processAsyncWorker(ctx, file, jobID, userID, totalRows)
return &ImportResult{ return &ImportResult{
JobID: jobID, JobID: jobID,
@ -252,9 +266,8 @@ func (i *Importer) processAsync(file io.Reader, userID string, totalRows int) (*
}, nil }, nil
} }
func (i *Importer) processAsyncWorker(file io.Reader, jobID string, userID string, totalRows int) { func (i *Importer) processAsyncWorker(ctx context.Context, file io.Reader, jobID string, userID string, totalRows int) {
db := i.Helper.GetDB("master") db := i.Helper.GetDB("master")
ctx := context.Background()
var imported, skipped int var imported, skipped int
var logs []ImportLog var logs []ImportLog
@ -263,9 +276,15 @@ func (i *Importer) processAsyncWorker(file io.Reader, jobID string, userID strin
var batchRows []int var batchRows []int
batchSize := 500 batchSize := 500
defer func() {
cancelMu.Lock()
delete(cancelFuncs, jobID)
cancelMu.Unlock()
}()
i.JobQ.UpdateProgress(ctx, jobID, 0, 0, nil) i.JobQ.UpdateProgress(ctx, jobID, 0, 0, nil)
csvlib.ReadCSV(file, func(row []string, rowNumber int) error { csvlib.ReadCSVWithContext(ctx, file, func(row []string, rowNumber int) error {
data, rowErrors := i.Handler.ValidateRow(row, rowNumber) data, rowErrors := i.Handler.ValidateRow(row, rowNumber)
if len(rowErrors) > 0 { if len(rowErrors) > 0 {
for _, e := range rowErrors { for _, e := range rowErrors {
@ -350,10 +369,27 @@ func (i *Importer) processAsyncWorker(file io.Reader, jobID string, userID strin
} }
} }
i.JobQ.CompleteJob(ctx, jobID, imported, skipped, errors) if ctx.Err() != nil {
i.JobQ.StopJob(ctx, jobID)
} else {
i.JobQ.CompleteJob(ctx, jobID, imported, skipped, errors)
}
i.saveJobLogs(ctx, jobID, logs, imported, skipped, errors) i.saveJobLogs(ctx, jobID, logs, imported, skipped, errors)
} }
func (i *Importer) StopJob(jobID string) error {
cancelMu.RLock()
cancel, exists := cancelFuncs[jobID]
cancelMu.RUnlock()
if !exists {
return fmt.Errorf("job not found or already completed")
}
cancel()
return nil
}
func (i *Importer) saveJobLogs(ctx context.Context, jobID string, logs []ImportLog, imported int, skipped int, errors []string) { func (i *Importer) saveJobLogs(ctx context.Context, jobID string, logs []ImportLog, imported int, skipped int, errors []string) {
logsJSON, _ := json.Marshal(logs) logsJSON, _ := json.Marshal(logs)
key := fmt.Sprintf("import_job:%s:logs", jobID) key := fmt.Sprintf("import_job:%s:logs", jobID)

View File

@ -15,6 +15,7 @@ const (
StatusProcessing JobStatus = "processing" StatusProcessing JobStatus = "processing"
StatusCompleted JobStatus = "completed" StatusCompleted JobStatus = "completed"
StatusFailed JobStatus = "failed" StatusFailed JobStatus = "failed"
StatusStopped JobStatus = "stopped"
) )
type Job struct { type Job struct {
@ -124,6 +125,16 @@ func (jq *JobQueue) FailJob(ctx context.Context, jobID string, errMsg string) er
}).Err() }).Err()
} }
func (jq *JobQueue) StopJob(ctx context.Context, jobID string) error {
key := fmt.Sprintf("import_job:%s", jobID)
now := time.Now()
return jq.Redis.HSet(ctx, key, map[string]interface{}{
"status": string(StatusStopped),
"finished_at": now.Format(time.RFC3339),
}).Err()
}
func (jq *JobQueue) GetJob(ctx context.Context, jobID string) (*Job, error) { func (jq *JobQueue) GetJob(ctx context.Context, jobID string) (*Job, error) {
key := fmt.Sprintf("import_job:%s", jobID) key := fmt.Sprintf("import_job:%s", jobID)
data, err := jq.Redis.HGetAll(ctx, key).Result() data, err := jq.Redis.HGetAll(ctx, key).Result()