Skip to content

Commit 88a5e7e

Browse files
ushareroseusharerose
committed
feat: collect Zentao Story-Repo-Commit from DB
1 parent 3a7acd6 commit 88a5e7e

File tree

2 files changed

+193
-2
lines changed

2 files changed

+193
-2
lines changed

‎backend/plugins/zentao/impl/impl.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,8 +153,7 @@ func (p Zentao) SubTaskMetas() []plugin.SubTaskMeta {
153153

154154
tasks.CollectStoryCommitsMeta,
155155
tasks.ExtractStoryCommitsMeta,
156-
tasks.CollectStoryRepoCommitsMeta,
157-
tasks.ExtractStoryRepoCommitsMeta,
156+
tasks.DBGetStoryRepoCommitsMeta,
158157
tasks.ConvertStoryRepoCommitsMeta,
159158

160159
tasks.CollectBugCommitsMeta,

‎backend/plugins/zentao/tasks/story_repo_commits_dbget.go

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package tasks
19+
20+
import (
21+
"encoding/json"
22+
"reflect"
23+
"strconv"
24+
25+
"github.com/apache/incubator-devlake/core/dal"
26+
"github.com/apache/incubator-devlake/core/errors"
27+
"github.com/apache/incubator-devlake/core/plugin"
28+
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
29+
"github.com/apache/incubator-devlake/plugins/zentao/models"
30+
)
31+
32+
var _ plugin.SubTaskEntryPoint = DBGetStoryRepoCommits
33+
34+
var DBGetStoryRepoCommitsMeta = plugin.SubTaskMeta{
35+
Name: "collectStoryRepoCommits",
36+
EntryPoint: DBGetStoryRepoCommits,
37+
EnabledByDefault: true,
38+
Description: "Get story commits data from Zentao database",
39+
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
40+
}
41+
42+
func DBGetStoryRepoCommits(taskCtx plugin.SubTaskContext) errors.Error {
43+
data := taskCtx.GetData().(*ZentaoTaskData)
44+
45+
// skip if no RemoteDb
46+
if data.RemoteDb == nil {
47+
return nil
48+
}
49+
50+
divider := api.NewBatchSaveDivider(taskCtx, 500, "", "")
51+
defer func() {
52+
err1 := divider.Close()
53+
if err1 != nil {
54+
panic(err1)
55+
}
56+
}()
57+
handler, err := newStoryRepoCommitHandler(taskCtx, divider)
58+
if err != nil {
59+
return err
60+
}
61+
return handler.collectStoryRepoCommit(
62+
taskCtx.GetDal(),
63+
data.RemoteDb,
64+
data.Options.ProjectId,
65+
data.Options.ConnectionId,
66+
)
67+
}
68+
69+
type storyRepoCommitHandler struct {
70+
rawDataParams string
71+
storyRepoCommitBachSave *api.BatchSave
72+
}
73+
74+
type toolZentaoStory struct {
75+
StoryID int64 `gorm:"type:integer"`
76+
}
77+
78+
type RemoteStoryRepoCommit struct {
79+
Project int64 `gorm:"type:integer"`
80+
IssueID int64 `gorm:"type:integer"`
81+
RepoUrl string
82+
CommitSha string
83+
}
84+
85+
func (h storyRepoCommitHandler) collectStoryRepoCommit(db dal.Dal, rdb dal.Dal, projectId int64, connectionId uint64) errors.Error {
86+
storyCursor, err := db.RawCursor(`
87+
SELECT
88+
DISTINCT fact_project_story.story_id AS story_id
89+
FROM
90+
_tool_zentao_project_stories AS fact_project_story
91+
LEFT JOIN
92+
_tool_zentao_stories AS fact_story
93+
ON
94+
fact_project_story.story_id = fact_story.id AND
95+
fact_project_story.connection_id = fact_story.connection_id
96+
WHERE
97+
fact_project_story.project_id = ? AND
98+
fact_project_story.connection_id = ?
99+
`, projectId, connectionId)
100+
if err != nil {
101+
return err
102+
}
103+
defer storyCursor.Close()
104+
var storyIds []int64
105+
for storyCursor.Next() {
106+
var row toolZentaoStory
107+
err := db.Fetch(storyCursor, &row)
108+
if err != nil {
109+
return errors.Default.Wrap(err, "error fetching storyCursor")
110+
}
111+
storyIds = append(storyIds, row.StoryID)
112+
}
113+
114+
remoteCursor, err := rdb.RawCursor(`
115+
SELECT
116+
DISTINCT dim_story_commits.story_id AS issue_id,
117+
dim_repo.path AS repo_url,
118+
dim_revision.revision AS commit_sha
119+
FROM (
120+
SELECT
121+
fact_action.objectID AS story_id,
122+
fact_action.project AS project_id,
123+
fact_action.product AS product_id,
124+
fact_action.extra AS short_commit_hexsha
125+
FROM
126+
zt_action AS fact_action
127+
WHERE
128+
fact_action.objectType IN ('story', 'requirement') AND
129+
fact_action.objectID IN ? AND
130+
fact_action.action IN ('gitcommited')
131+
) AS dim_story_commits
132+
INNER JOIN (
133+
SELECT
134+
fact_repo_hist.repo AS repo_id,
135+
fact_repo_hist.revision AS revision,
136+
LEFT(fact_repo_hist.revision, 10) AS short_commit_hexsha
137+
FROM
138+
zt_repohistory AS fact_repo_hist
139+
) AS dim_revision
140+
ON
141+
dim_story_commits.short_commit_hexsha = dim_revision.short_commit_hexsha
142+
INNER JOIN
143+
zt_repo AS dim_repo
144+
ON
145+
dim_revision.repo_id = dim_repo.id
146+
`, storyIds)
147+
if err != nil {
148+
return err
149+
}
150+
defer remoteCursor.Close()
151+
152+
for remoteCursor.Next() {
153+
var remoteStoryRepoCommit RemoteStoryRepoCommit
154+
err = rdb.Fetch(remoteCursor, &remoteStoryRepoCommit)
155+
if err != nil {
156+
return err
157+
}
158+
storyRepoCommit := &models.ZentaoStoryRepoCommit{
159+
ConnectionId: connectionId,
160+
Project: projectId,
161+
RepoUrl: remoteStoryRepoCommit.RepoUrl,
162+
CommitSha: remoteStoryRepoCommit.CommitSha,
163+
IssueId: strconv.FormatInt(remoteStoryRepoCommit.IssueID, 10),
164+
}
165+
storyRepoCommit.NoPKModel.RawDataParams = h.rawDataParams
166+
err = h.storyRepoCommitBachSave.Add(storyRepoCommit)
167+
if err != nil {
168+
return err
169+
}
170+
}
171+
return h.storyRepoCommitBachSave.Flush()
172+
}
173+
174+
func newStoryRepoCommitHandler(taskCtx plugin.SubTaskContext, divider *api.BatchSaveDivider) (*storyRepoCommitHandler, errors.Error) {
175+
data := taskCtx.GetData().(*ZentaoTaskData)
176+
177+
storyRepoCommitBachSave, err := divider.ForType(reflect.TypeOf(&models.ZentaoStoryRepoCommit{}))
178+
if err != nil {
179+
return nil, err
180+
}
181+
blob, _ := json.Marshal(data.Options.GetParams())
182+
rawDataParams := string(blob)
183+
db := taskCtx.GetDal()
184+
err = db.Delete(&models.ZentaoStoryRepoCommit{}, dal.Where("_raw_data_params = ?", rawDataParams))
185+
if err != nil {
186+
return nil, err
187+
}
188+
return &storyRepoCommitHandler{
189+
rawDataParams: rawDataParams,
190+
storyRepoCommitBachSave: storyRepoCommitBachSave,
191+
}, nil
192+
}

0 commit comments

Comments
 (0)