Skip to content

Commit ae4d8b9

Browse files
d4x1usharerose
d4x1
and
usharerose
authored
cherry pick #8298 and #8300 to v1.0 (#8301)
* feat: support Zentao task worklogs (#8298) * feat: support Zentao task worklogs * feat: Zentao worklog tool layer migration * fix: dict as raw data instead of slice * test: e2e for zentao task worklogs * fix: compatibility when no account available --------- Co-authored-by: Lynwee <1507509064@qq.com> * fix: go lint and import errors (#8300) * fix: import errors * fix: go fmt issues * fix: missing quote * fix: skip float accuracy issues * fix(zentao): fix e2e tests --------- Co-authored-by: Chaojie Yan <chaojie.yan@merico.dev>
1 parent 0ec948b commit ae4d8b9

12 files changed

+562
-0
lines changed

‎backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_task_worklogs.csv

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
id,params,data,url,input,created_at
2+
1,"{""ConnectionId"":1,""ProjectId"":48}","{""id"":106,""objectType"":""task"",""objectID"":135,""product"":"""",""project"":48,""execution"":49,""account"":""devlake"",""work"":""sample worklog"",""vision"":""rnd"",""date"":""2025-02-20"",""left"":5,""consumed"":11,""begin"":0,""end"":0,""extra"":null,""order"":0,""deleted"":""0""}",http://iwater.red:8000/api.php/v1/tasks/135/estimate,"{""id"":135}",2025-02-21 06:28:36.902
3+
2,"{""ConnectionId"":1,""ProjectId"":48}","{""id"":107,""objectType"":""task"",""objectID"":135,""product"":"""",""project"":48,""execution"":49,""account"":""devlake"",""work"":"""",""vision"":""rnd"",""date"":""2025-02-20"",""left"":1,""consumed"":4,""begin"":0,""end"":0,""extra"":null,""order"":0,""deleted"":""0""}",http://iwater.red:8000/api.php/v1/tasks/135/estimate,"{""id"":135}",2025-02-21 06:28:37.001

‎backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_worklogs.csv

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
connection_id,id,object_id,object_type,project,execution,product,account,work,vision,date,left,consumed,begin,end,extra,order,deleted
2+
1,106,135,task,48,49,,devlake,sample worklog,rnd,2025-02-20,5,11,0,0,,0,0
3+
1,107,135,task,48,49,,devlake,,rnd,2025-02-20,1,4,0,0,,0,0

‎backend/plugins/zentao/e2e/snapshot_tables/issue_worklogs.csv

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
id,author_id,comment,time_spent_minutes,logged_date,started_date,issue_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
2+
zentao:ZentaoWorklog:1:106,zentao:ZentaoAccount:1:1,sample worklog,660,2025-02-20T00:00:00.000+00:00,2025-02-20T00:00:00.000+00:00,zentao:ZentaoTask:1:135,"{""ConnectionId"":1,""ProjectId"":48}",_raw_zentao_api_task_worklogs,1,
3+
zentao:ZentaoWorklog:1:107,zentao:ZentaoAccount:1:1,,240,2025-02-20T00:00:00.000+00:00,2025-02-20T00:00:00.000+00:00,zentao:ZentaoTask:1:135,"{""ConnectionId"":1,""ProjectId"":48}",_raw_zentao_api_task_worklogs,2,

‎backend/plugins/zentao/e2e/task_worklogs_test.go

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package e2e
19+
20+
import (
21+
"testing"
22+
23+
"github.com/apache/incubator-devlake/core/models/common"
24+
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
25+
"github.com/apache/incubator-devlake/helpers/e2ehelper"
26+
"github.com/apache/incubator-devlake/plugins/zentao/impl"
27+
"github.com/apache/incubator-devlake/plugins/zentao/models"
28+
"github.com/apache/incubator-devlake/plugins/zentao/tasks"
29+
)
30+
31+
func TestZentaoTaskWorklogDataFlow(t *testing.T) {
32+
33+
var zentao impl.Zentao
34+
dataflowTester := e2ehelper.NewDataFlowTester(t, "zentao", zentao)
35+
36+
taskData := &tasks.ZentaoTaskData{
37+
Options: &tasks.ZentaoOptions{
38+
ConnectionId: 1,
39+
ProjectId: 48,
40+
},
41+
ApiClient: getFakeAPIClient(),
42+
}
43+
44+
// import _raw_zentao_api_task_worklogs raw data table
45+
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_zentao_api_task_worklogs.csv",
46+
"_raw_zentao_api_task_worklogs")
47+
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_zentao_accounts.csv", &models.ZentaoAccount{})
48+
49+
// verify worklogs extraction
50+
dataflowTester.FlushTabler(&models.ZentaoWorklog{})
51+
dataflowTester.Subtask(tasks.ExtractTaskWorklogsMeta, taskData)
52+
dataflowTester.VerifyTableWithOptions(&models.ZentaoWorklog{}, e2ehelper.TableOptions{
53+
CSVRelPath: "./snapshot_tables/_tool_zentao_worklogs.csv",
54+
IgnoreTypes: []interface{}{common.NoPKModel{}},
55+
})
56+
57+
// verify task repo commit conversion
58+
dataflowTester.FlushTabler(&ticket.IssueWorklog{})
59+
dataflowTester.Subtask(tasks.ConvertTaskWorklogsMeta, taskData)
60+
dataflowTester.VerifyTableWithOptions(&ticket.IssueWorklog{}, e2ehelper.TableOptions{
61+
CSVRelPath: "./snapshot_tables/issue_worklogs.csv",
62+
IgnoreTypes: []interface{}{common.NoPKModel{}},
63+
})
64+
}

‎backend/plugins/zentao/impl/impl.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ func (p Zentao) GetTablesInfo() []dal.Tabler {
9191
&models.ZentaoExecutionSummary{},
9292
&models.ZentaoProductSummary{},
9393
&models.ZentaoProjectStory{},
94+
&models.ZentaoWorklog{},
9495
}
9596
}
9697

@@ -160,6 +161,10 @@ func (p Zentao) SubTaskMetas() []plugin.SubTaskMeta {
160161

161162
tasks.DBGetChangelogMeta,
162163
tasks.ConvertChangelogMeta,
164+
165+
tasks.CollectTaskWorklogsMeta,
166+
tasks.ExtractTaskWorklogsMeta,
167+
tasks.ConvertTaskWorklogsMeta,
163168
}
164169
}
165170

‎backend/plugins/zentao/models/migrationscripts/20250219_add_worklogs.go

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package migrationscripts
19+
20+
import (
21+
"github.com/apache/incubator-devlake/core/context"
22+
"github.com/apache/incubator-devlake/core/errors"
23+
"github.com/apache/incubator-devlake/helpers/migrationhelper"
24+
"github.com/apache/incubator-devlake/plugins/zentao/models/migrationscripts/archived"
25+
)
26+
27+
type addWorklogs struct{}
28+
29+
func (*addWorklogs) Up(basicRes context.BasicRes) errors.Error {
30+
return migrationhelper.AutoMigrateTables(
31+
basicRes,
32+
&archived.ZentaoWorklog{},
33+
)
34+
}
35+
36+
func (*addWorklogs) Version() uint64 {
37+
return 20250219153329
38+
}
39+
40+
func (*addWorklogs) Name() string {
41+
return "add table _tool_zentao_worklogs"
42+
}

‎backend/plugins/zentao/models/migrationscripts/archived/worklog.go

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package archived
19+
20+
import (
21+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
22+
)
23+
24+
type ZentaoWorklog struct {
25+
archived.NoPKModel
26+
ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
27+
Id uint64 `gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false" json:"id"`
28+
ObjectId uint64 `json:"objectID"`
29+
ObjectType string `json:"objectType"`
30+
Project uint64 `json:"project"`
31+
Execution uint64 `json:"execution"`
32+
Product string `json:"product"`
33+
Account string `json:"account"`
34+
Work string `json:"work"`
35+
Vision string `json:"vision"`
36+
Date string `json:"date"`
37+
Left float32 `json:"left"`
38+
Consumed float32 `json:"consumed"`
39+
Begin uint64 `json:"begin"`
40+
End uint64 `json:"end"`
41+
Extra *string `json:"extra"`
42+
Order uint64 `json:"order"`
43+
Deleted string `json:"deleted"`
44+
}
45+
46+
func (ZentaoWorklog) TableName() string {
47+
return "_tool_zentao_worklogs"
48+
}

‎backend/plugins/zentao/models/migrationscripts/register.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,5 +32,6 @@ func All() []plugin.MigrationScript {
3232
new(addExecutionStoryAndExecutionSummary),
3333
new(addRawParamTableForScope),
3434
new(dropTotalReal),
35+
new(addWorklogs),
3536
}
3637
}

‎backend/plugins/zentao/models/worklog.go

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package models
19+
20+
import (
21+
"github.com/apache/incubator-devlake/core/models/common"
22+
)
23+
24+
type ZentaoWorklog struct {
25+
ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
26+
Id int64 `gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false" json:"id"`
27+
ObjectId int64 `json:"objectID"`
28+
ObjectType string `json:"objectType"`
29+
Project int64 `json:"project"`
30+
Execution int64 `json:"execution"`
31+
Product string `json:"product"`
32+
Account string `json:"account"`
33+
Work string `json:"work"`
34+
Vision string `json:"vision"`
35+
Date string `json:"date"`
36+
Left float32 `json:"left"`
37+
Consumed float32 `json:"consumed"`
38+
Begin int64 `json:"begin"`
39+
End int64 `json:"end"`
40+
Extra *string `json:"extra"`
41+
Order int64 `json:"order"`
42+
Deleted string `json:"deleted"`
43+
common.NoPKModel
44+
}
45+
46+
func (ZentaoWorklog) TableName() string {
47+
return "_tool_zentao_worklogs"
48+
}

‎backend/plugins/zentao/tasks/task_worklog_collector.go

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package tasks
19+
20+
import (
21+
"encoding/json"
22+
"net/http"
23+
"net/url"
24+
"reflect"
25+
26+
"github.com/apache/incubator-devlake/core/dal"
27+
"github.com/apache/incubator-devlake/core/errors"
28+
"github.com/apache/incubator-devlake/core/plugin"
29+
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
30+
"github.com/apache/incubator-devlake/plugins/zentao/models"
31+
)
32+
33+
const RAW_TASK_WORKLOGS_TABLE = "zentao_api_task_worklogs"
34+
35+
var CollectTaskWorklogsMeta = plugin.SubTaskMeta{
36+
Name: "collectTaskWorklogs",
37+
EntryPoint: CollectTaskWorklogs,
38+
EnabledByDefault: true,
39+
Description: "collect Zentao task work logs, supports both timeFilter and diffSync.",
40+
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
41+
}
42+
43+
type Input struct {
44+
Id uint64 `json:"id"`
45+
}
46+
47+
func CollectTaskWorklogs(taskCtx plugin.SubTaskContext) errors.Error {
48+
db := taskCtx.GetDal()
49+
data := taskCtx.GetData().(*ZentaoTaskData)
50+
51+
logger := taskCtx.GetLogger()
52+
53+
apiCollector, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{
54+
Ctx: taskCtx,
55+
Options: data.Options,
56+
Table: RAW_TASK_WORKLOGS_TABLE,
57+
})
58+
if err != nil {
59+
return err
60+
}
61+
62+
// load task IDs from db
63+
clauses := []dal.Clause{
64+
dal.Select("id"),
65+
dal.From(&models.ZentaoTask{}),
66+
dal.Where(
67+
"project = ? AND connection_id = ?",
68+
data.Options.ProjectId, data.Options.ConnectionId,
69+
),
70+
}
71+
if apiCollector.IsIncremental() && apiCollector.GetSince() != nil {
72+
clauses = append(clauses, dal.Where("last_edited_date IS NOT NULL AND last_edited_date > ?", apiCollector.GetSince()))
73+
}
74+
75+
// construct the input iterator
76+
cursor, err := db.Cursor(clauses...)
77+
if err != nil {
78+
return err
79+
}
80+
iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(Input{}))
81+
if err != nil {
82+
return err
83+
}
84+
85+
// collect task worklogs
86+
err = apiCollector.InitCollector(api.ApiCollectorArgs{
87+
Input: iterator,
88+
ApiClient: data.ApiClient,
89+
UrlTemplate: "tasks/{{ .Input.Id }}/estimate",
90+
Query: func(reqData *api.RequestData) (url.Values, errors.Error) {
91+
return nil, nil
92+
},
93+
ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
94+
var data struct {
95+
Effort json.RawMessage `json:"effort"`
96+
}
97+
err := api.UnmarshalResponse(res, &data)
98+
if err != nil {
99+
return nil, err
100+
}
101+
102+
if string(data.Effort) == "{}" || string(data.Effort) == "null" {
103+
return nil, nil
104+
}
105+
106+
var efforts []json.RawMessage
107+
jsonErr := json.Unmarshal(data.Effort, &efforts)
108+
if jsonErr != nil {
109+
return nil, errors.Default.Wrap(jsonErr, "failed to unmarshal efforts")
110+
}
111+
return efforts, nil
112+
},
113+
AfterResponse: ignoreHTTPStatus404,
114+
})
115+
if err != nil {
116+
logger.Error(err, "collect Zentao task worklogs error")
117+
return err
118+
}
119+
120+
return apiCollector.Execute()
121+
}

0 commit comments

Comments
 (0)