Skip to content

[update] modify the RefreshAgents interface to use batch operations; #922

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[update] modify the RefreshAgents interface to use batch operations; #922

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
List<User> GetUsersByAffiliateId(string affiliateId) => throw new NotImplementedException();
User? GetUserByUserName(string userName) => throw new NotImplementedException();
void UpdateUserName(string userId, string userName) => throw new NotImplementedException();
Dashboard? GetDashboard(string id = null) => throw new NotImplementedException();

Check warning on line 47 in src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

View workflow job for this annotation

GitHub Actions / build (macos-latest) 

Cannot convert null literal to non-nullable reference type.

Check warning on line 47 in src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

View workflow job for this annotation

GitHub Actions / build (macos-latest) 

Cannot convert null literal to non-nullable reference type.

Check warning on line 47 in src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

View workflow job for this annotation

GitHub Actions / build (ubuntu-latest) 

Cannot convert null literal to non-nullable reference type.

Check warning on line 47 in src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

View workflow job for this annotation

GitHub Actions / build (ubuntu-latest) 

Cannot convert null literal to non-nullable reference type.

Check warning on line 47 in src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

View workflow job for this annotation

GitHub Actions / build (windows-latest) 

Cannot convert null literal to non-nullable reference type.

Check warning on line 47 in src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

View workflow job for this annotation

GitHub Actions / build (windows-latest) 

Cannot convert null literal to non-nullable reference type.
void CreateUser(User user) => throw new NotImplementedException();
void UpdateExistUser(string userId, User user) => throw new NotImplementedException();
void UpdateUserVerified(string userId) => throw new NotImplementedException();
Expand Down Expand Up @@ -74,10 +74,18 @@
=> throw new NotImplementedException();
void BulkInsertAgents(List<Agent> agents)
=> throw new NotImplementedException();
ValueTask BulkInsertAgentsAsync(List<Agent> agents)
=> throw new NotImplementedException();
void BulkInsertUserAgents(List<UserAgent> userAgents)
=> throw new NotImplementedException();
ValueTask BulkInsertUserAgentsAsync(List<UserAgent> userAgents)
=> throw new NotImplementedException();
bool DeleteAgents()
=> throw new NotImplementedException();
Task<bool> DeleteAgentsAsync()
=> throw new NotImplementedException();
ValueTask<bool> DeleteAgentsAsync(List<string> agentIds)
=> throw new NotImplementedException();
bool DeleteAgent(string agentId)
=> throw new NotImplementedException();
List<string> GetAgentResponses(string agentId, string prefix, string intent)
Expand All @@ -102,6 +110,8 @@
=> throw new NotImplementedException();
void BulkInsertAgentTasks(List<AgentTask> tasks)
=> throw new NotImplementedException();
ValueTask BulkInsertAgentTasksAsync(List<AgentTask> tasks)
=> throw new NotImplementedException();
void UpdateAgentTask(AgentTask task, AgentTaskField field)
=> throw new NotImplementedException();
bool DeleteAgentTask(string agentId, List<string> taskIds)
Expand Down
73 changes: 51 additions & 22 deletions src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using BotSharp.Abstraction.Repositories.Enums;
using BotSharp.Abstraction.Tasks.Models;
using System.IO;

namespace BotSharp.Core.Agents.Services;
Expand All @@ -24,27 +25,24 @@ public async Task<string> RefreshAgents()
return "Unauthorized user.";
}

var agentDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory,
dbSettings.FileRepository,
_agentSettings.DataDir);

var agentDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, dbSettings.FileRepository, _agentSettings.DataDir);
if (!Directory.Exists(agentDir))
{
refreshResult = $"Cannot find the directory: {agentDir}";
return refreshResult;
}

var refreshedAgents = new List<string>();

List<Agent> agents = [];
List<AgentTask> agentTasks = [];

foreach (var dir in Directory.GetDirectories(agentDir))
{
try
{
var agentJson = File.ReadAllText(Path.Combine(dir, "agent.json"));
var agent = JsonSerializer.Deserialize<Agent>(agentJson, _options);

var (agent, msg) = GetAgentFormJson(Path.Combine(dir, "agent.json"));
if (agent == null)
{
_logger.LogError($"Cannot find agent in file directory: {dir}");
_logger.LogError(msg);
continue;
}

Expand All @@ -61,27 +59,26 @@ public async Task<string> RefreshAgents()
.SetSamples(samples);

var tasks = GetTasksFromFile(dir);

var isAgentDeleted = _db.DeleteAgent(agent.Id);
if (isAgentDeleted)
{
await Task.Delay(100);
_db.BulkInsertAgents(new List<Agent> { agent });
_db.BulkInsertAgentTasks(tasks);
refreshedAgents.Add(agent.Name);
_logger.LogInformation($"Agent {agent.Name} has been migrated.");
}
if (!tasks.IsNullOrEmpty()) agentTasks.AddRange(tasks);
agents.Add(agent);
}
catch (Exception ex)
{
_logger.LogError($"Failed to migrate agent in file directory: {dir}\r\nError: {ex.Message}");
}
}

if (!refreshedAgents.IsNullOrEmpty())
if (agents.Count > 0)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please ensure that when performing batch operations, the data operations are carried out within a single transaction in MongoDB to prevent other agents from failing to insert when one agent insertion encounters an exception.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  • IsNullOrEmpty is a built-in extension method.
  • Okay, batch operations can adjust transaction support.

Copy link
Contributor Author

@zfchai zfchai Mar 12, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The IBotSharpRepository currently does not provide transaction session objects.
If you consider multiple database support, this session object may not be compatible.

{
var agentIds = agents.Select(a => a.Id).ToList();
await _db.DeleteAgentsAsync(agentIds);
await Task.Delay(200);
await _db.BulkInsertAgentsAsync(agents);
await Task.Delay(200);
await _db.BulkInsertAgentTasksAsync(agentTasks);

Utilities.ClearCache();
refreshResult = $"Agents are migrated!\r\n{string.Join("\r\n", refreshedAgents)}";
refreshResult = $"Agents are migrated!\r\n{string.Join("\r\n", agents.Select(a => a.Name))}";
}
else
{
Expand All @@ -91,4 +88,36 @@ public async Task<string> RefreshAgents()
_logger.LogInformation(refreshResult);
return refreshResult;
}

private (Agent? agent, string msg) GetAgentFormJson(string agentPath)
{
var agentJson = File.ReadAllText(agentPath);
if (string.IsNullOrWhiteSpace(agentJson))
return (null, $"Cannot find agent in file path: {agentPath}");

var isJson = IsValidedJson(agentJson);
if (isJson)
{
var agent = JsonSerializer.Deserialize<Agent>(agentJson, _options);
return (agent, "ok");
}
else
{
return (null, "The agent.json file data is not in JSON format!");
}
}

private bool IsValidedJson(string jsonString)
{
try
{
JsonDocument.Parse(jsonString);
return true;
}
catch (JsonException ex)
{
return false;
}
}

}
30 changes: 30 additions & 0 deletions src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -582,6 +582,12 @@ public void BulkInsertAgents(List<Agent> agents)
ResetInnerAgents();
}

public async ValueTask BulkInsertAgentsAsync(List<Agent> agents)
{
await Task.Delay(100);
BulkInsertAgents(agents);
}

public void BulkInsertUserAgents(List<UserAgent> userAgents)
{
if (userAgents.IsNullOrEmpty()) return;
Expand Down Expand Up @@ -615,11 +621,35 @@ public void BulkInsertUserAgents(List<UserAgent> userAgents)
ResetInnerAgents();
}

public async ValueTask BulkInsertUserAgentsAsync(List<UserAgent> userAgents)
{
await Task.Delay(200);
BulkInsertUserAgents(userAgents);
}

public bool DeleteAgents()
{
return false;
}

public async Task<bool> DeleteAgentsAsync()
{
await Task.Delay(100);
return false;
}

public async ValueTask<bool> DeleteAgentsAsync(List<string> agentIds)
{
bool isDelete = false;
foreach (var agentId in agentIds)
{
isDelete = DeleteAgent(agentId);
await Task.Delay(200);
}

return isDelete;
}

public bool DeleteAgent(string agentId)
{
if (string.IsNullOrEmpty(agentId)) return false;
Expand Down
7 changes: 7 additions & 0 deletions src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,13 @@ public void BulkInsertAgentTasks(List<AgentTask> tasks)

}

public async ValueTask BulkInsertAgentTasksAsync(List<AgentTask> tasks)
{
if (tasks.IsNullOrEmpty()) return;

await Task.Delay(200);
}

public void UpdateAgentTask(AgentTask task, AgentTaskField field)
{
if (task == null || string.IsNullOrEmpty(task.Id)) return;
Expand Down
Loading
Loading