Skip to content
This repository was archived by the owner on Aug 10, 2022. It is now read-only.

Commit 58ac1d5

Browse files
TheCodeTravelerTheCodeTraveler
authored
Merge pull request #24 from brminnick/Optimize-GetMicrosoftLearnContributors-API
Optimize GetMicrosoftLearnContributors API
2 parents 729ce8b + 9c1f47a commit 58ac1d5

8 files changed

Lines changed: 208 additions & 61 deletions

File tree

‎AzureAdvocates.Functions/AzureAdvocates.Functions.csproj‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,17 @@
33
<TargetFramework>netcoreapp3.1</TargetFramework>
44
<AzureFunctionsVersion>v3</AzureFunctionsVersion>
55
<LangVersion>latest</LangVersion>
6+
<Nullable>enable</Nullable>
67
</PropertyGroup>
78
<ItemGroup>
89
<PackageReference Include="GitHubApiStatus.Extensions" Version="2.0.0-pre9" />
910
<PackageReference Include="Microsoft.Azure.Functions.Extensions" Version="1.1.0" />
1011
<PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.11" />
1112
</ItemGroup>
1213
<ItemGroup>
13-
<ProjectReference Include="..\GitHubReadmeWebTrends.Common\GitHubReadmeWebTrends.Common.csproj" />
14+
<ProjectReference Include="..\GitHubReadmeWebTrends.Common\GitHubReadmeWebTrends.Common.csproj">
15+
<GlobalPropertiesToRemove></GlobalPropertiesToRemove>
16+
</ProjectReference>
1417
</ItemGroup>
1518
<ItemGroup>
1619
<None Update="host.json">

‎AzureAdvocates.Functions/Functions/GetMicrosoftLearnContributors.cs‎

Lines changed: 13 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,80 +1,43 @@
11
using System;
22
using System.Collections.Generic;
33
using System.Linq;
4-
using System.Net;
54
using System.Net.Http;
6-
using System.Net.Http.Headers;
7-
using System.Threading;
85
using System.Threading.Tasks;
9-
using GitHubApiStatus;
10-
using GitHubReadmeWebTrends.Common;
116
using Microsoft.AspNetCore.Mvc;
127
using Microsoft.Azure.WebJobs;
138
using Microsoft.Azure.WebJobs.Extensions.Http;
149
using Microsoft.Extensions.Logging;
1510

16-
namespace GitHubReadmeWebTrends.Functions
11+
namespace AzureAdvocates.Functions
1712
{
1813
class GetMicrosoftLearnContributors
1914
{
20-
const string _defaultTeam = "";
15+
readonly BlobStorageService _blobStorageService;
2116

22-
readonly CloudAdvocateService _cloudAdvocateService;
23-
readonly IGitHubApiStatusService _gitHubApiStatusService;
24-
readonly GitHubGraphQLApiService _gitHubGraphQLApiService;
25-
26-
public GetMicrosoftLearnContributors(CloudAdvocateService cloudAdvocateService,
27-
IGitHubApiStatusService gitHubApiStatusService,
28-
GitHubGraphQLApiService gitHubGraphQLApiService)
29-
{
30-
_cloudAdvocateService = cloudAdvocateService;
31-
_gitHubApiStatusService = gitHubApiStatusService;
32-
_gitHubGraphQLApiService = gitHubGraphQLApiService;
33-
}
17+
public GetMicrosoftLearnContributors(BlobStorageService blobStorageService) => _blobStorageService = blobStorageService;
3418

3519
[FunctionName(nameof(GetMicrosoftLearnContributors))]
3620
public async Task<IActionResult> Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = nameof(GetMicrosoftLearnContributors) + "/{from:datetime}/{to:datetime}/{team?}")] HttpRequestMessage req, DateTime from, DateTime to, string? team, ILogger log)
3721
{
3822
log.LogInformation($"{nameof(GetMicrosoftLearnContributors)} Started");
3923

40-
var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(2));
41-
var gitHubApiStatus = await _gitHubApiStatusService.GetApiRateLimits(cancellationTokenSource.Token).ConfigureAwait(false);
42-
if (gitHubApiStatus.GraphQLApi.RemainingRequestCount < 4000)
43-
{
44-
return new ObjectResult($"Maximum GitHub API Limit Reached. GitHub API Limit will reset in {gitHubApiStatus.GraphQLApi.RateLimitReset_TimeRemaining.Minutes + 1} minute(s). Try again at {gitHubApiStatus.GraphQLApi.RateLimitReset_DateTime.UtcDateTime} UTC")
45-
{
46-
StatusCode = (int)HttpStatusCode.Forbidden
47-
};
48-
}
24+
var microsoftLearnContributionsList = await _blobStorageService.GetCloudAdvocateMicrosoftLearnContributors().ConfigureAwait(false);
4925

50-
var cloudAdvocateList = new List<CloudAdvocateGitHubUserModel>();
51-
await foreach (var advocate in _cloudAdvocateService.GetAzureAdvocates().ConfigureAwait(false))
26+
var filteredCloudAdvocateContributions = new List<CloudAdvocateGitHubContributorModel>();
27+
foreach (var advocateContribution in microsoftLearnContributionsList)
5228
{
53-
if (team is null || advocate.MicrosoftTeam.Equals(team, StringComparison.OrdinalIgnoreCase))
29+
if (team is null || advocateContribution.MicrosoftTeam.Equals(team, StringComparison.OrdinalIgnoreCase))
5430
{
55-
log.LogInformation($"Found Advocate: {advocate.FullName}");
56-
cloudAdvocateList.Add(advocate);
57-
}
58-
}
31+
log.LogInformation($"Adding Advocate: {advocateContribution.FullName}");
5932

60-
var microsoftLearnPullRequests = new List<RepositoryPullRequest>();
61-
await foreach (var pullRequestList in _gitHubGraphQLApiService.GetMicrosoftLearnPullRequests().ConfigureAwait(false))
62-
{
63-
microsoftLearnPullRequests.AddRange(pullRequestList);
64-
log.LogInformation($"Added {pullRequestList.Count} Pull Requests from {pullRequestList.FirstOrDefault()?.RepositoryName}");
65-
}
33+
var filteredPullRequests = advocateContribution.PullRequests.Where(x => x.CreatedAt.IsWithinRange(from, to)).ToList();
34+
var filteredCloudAdvocateContribution = new CloudAdvocateGitHubContributorModel(filteredPullRequests, advocateContribution.FullName, advocateContribution.GitHubUserName, advocateContribution.MicrosoftAlias, advocateContribution.MicrosoftTeam);
6635

67-
var cloudAdvocateContributions = new List<GitHubContributorModel>();
68-
foreach (var cloudAdvocate in cloudAdvocateList)
69-
{
70-
var cloudAdvocateContributorModel = new GitHubContributorModel(microsoftLearnPullRequests.Where(x => x.Author.Equals(cloudAdvocate.GitHubUserName, StringComparison.OrdinalIgnoreCase) && x.CreatedAt.IsWithinRange(from, to)), cloudAdvocate);
71-
72-
cloudAdvocateContributions.Add(cloudAdvocateContributorModel);
73-
74-
log.LogInformation($"Added {cloudAdvocateContributorModel.PullRequests.Count} Pull Requests for {cloudAdvocate.FullName}");
36+
filteredCloudAdvocateContributions.Add(filteredCloudAdvocateContribution);
37+
}
7538
}
7639

77-
return new OkObjectResult(cloudAdvocateContributions);
40+
return new OkObjectResult(filteredCloudAdvocateContributions);
7841
}
7942
}
8043

‎AzureAdvocates.Functions/Functions/UpdateMicrosoftLearnContributors.cs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Threading;
5+
using System.Threading.Tasks;
6+
using GitHubApiStatus;
7+
using GitHubReadmeWebTrends.Common;
8+
using Microsoft.Azure.WebJobs;
9+
using Microsoft.Extensions.Logging;
10+
11+
namespace AzureAdvocates.Functions
12+
{
13+
class UpdateMicrosoftLearnContributors
14+
{
15+
readonly BlobStorageService _blobStorageService;
16+
readonly CloudAdvocateService _cloudAdvocateService;
17+
readonly IGitHubApiStatusService _gitHubApiStatusService;
18+
readonly GitHubGraphQLApiService _gitHubGraphQLApiService;
19+
20+
public UpdateMicrosoftLearnContributors(BlobStorageService blobStorageService,
21+
CloudAdvocateService cloudAdvocateService,
22+
IGitHubApiStatusService gitHubApiStatusService,
23+
GitHubGraphQLApiService gitHubGraphQLApiService)
24+
{
25+
_blobStorageService = blobStorageService;
26+
_cloudAdvocateService = cloudAdvocateService;
27+
_gitHubApiStatusService = gitHubApiStatusService;
28+
_gitHubGraphQLApiService = gitHubGraphQLApiService;
29+
}
30+
31+
[FunctionName(nameof(UpdateMicrosoftLearnContributors))]
32+
public async Task Run([TimerTrigger("0 0 */6 * * *")] TimerInfo timer, ILogger log)
33+
{
34+
log.LogInformation($"{nameof(UpdateMicrosoftLearnContributors)} Started");
35+
36+
var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(2));
37+
var gitHubApiStatus = await _gitHubApiStatusService.GetApiRateLimits(cancellationTokenSource.Token).ConfigureAwait(false);
38+
if (gitHubApiStatus.GraphQLApi.RemainingRequestCount < 4000)
39+
{
40+
log.LogError($"Maximum GitHub API Limit Reached. GitHub API Limit will reset in {gitHubApiStatus.GraphQLApi.RateLimitReset_TimeRemaining.Minutes + 1} minute(s). Try again at {gitHubApiStatus.GraphQLApi.RateLimitReset_DateTime.UtcDateTime} UTC");
41+
return;
42+
}
43+
44+
var cloudAdvocateList = new List<CloudAdvocateGitHubUserModel>();
45+
await foreach (var advocate in _cloudAdvocateService.GetAzureAdvocates().ConfigureAwait(false))
46+
{
47+
cloudAdvocateList.Add(advocate);
48+
}
49+
50+
var microsoftLearnPullRequests = new List<RepositoryPullRequest>();
51+
await foreach (var pullRequestList in _gitHubGraphQLApiService.GetMicrosoftLearnPullRequests().ConfigureAwait(false))
52+
{
53+
microsoftLearnPullRequests.AddRange(pullRequestList);
54+
log.LogInformation($"Added {pullRequestList.Count} Pull Requests from {pullRequestList.FirstOrDefault()?.RepositoryName}");
55+
}
56+
57+
var cloudAdvocateContributions = new List<CloudAdvocateGitHubContributorModel>();
58+
foreach (var cloudAdvocate in cloudAdvocateList)
59+
{
60+
var cloudAdvocateContributorModel = new CloudAdvocateGitHubContributorModel(microsoftLearnPullRequests.Where(x => x.Author.Equals(cloudAdvocate.GitHubUserName, StringComparison.OrdinalIgnoreCase)), cloudAdvocate);
61+
62+
cloudAdvocateContributions.Add(cloudAdvocateContributorModel);
63+
64+
log.LogInformation($"Added {cloudAdvocateContributorModel.PullRequests.Count} Pull Requests for {cloudAdvocate.FullName}");
65+
}
66+
67+
var blobName = $"Contributions_{DateTime.UtcNow:o}.json";
68+
await _blobStorageService.UploadCloudAdvocateMicrosoftLearnContributions(cloudAdvocateContributions, blobName).ConfigureAwait(false);
69+
}
70+
}
71+
}

‎AzureAdvocates.Functions/Models/GitHubContributorModel.cs‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,26 @@
11
using System.Collections.Generic;
22
using System.Linq;
33
using GitHubReadmeWebTrends.Common;
4+
using Newtonsoft.Json;
45

5-
namespace GitHubReadmeWebTrends.Functions
6+
namespace AzureAdvocates.Functions
67
{
7-
class GitHubContributorModel : CloudAdvocateGitHubUserModel
8+
class CloudAdvocateGitHubContributorModel : CloudAdvocateGitHubUserModel
89
{
9-
public GitHubContributorModel(in IEnumerable<RepositoryPullRequest> pullReuests, CloudAdvocateGitHubUserModel cloudAdvocateGitHubUserModel)
10-
: this(pullReuests, cloudAdvocateGitHubUserModel.FullName, cloudAdvocateGitHubUserModel.GitHubUserName, cloudAdvocateGitHubUserModel.MicrosoftAlias, cloudAdvocateGitHubUserModel.MicrosoftTeam)
10+
public CloudAdvocateGitHubContributorModel(IEnumerable<RepositoryPullRequest> pullRequests, CloudAdvocateGitHubUserModel cloudAdvocateGitHubUserModel)
11+
: this(pullRequests, cloudAdvocateGitHubUserModel.FullName, cloudAdvocateGitHubUserModel.GitHubUserName, cloudAdvocateGitHubUserModel.MicrosoftAlias, cloudAdvocateGitHubUserModel.MicrosoftTeam)
1112
{
1213

1314
}
1415

15-
public GitHubContributorModel(in IEnumerable<RepositoryPullRequest> pullReuests, in string fullName, in string gitHubUserName, in string microsoftAlias, in string microsoftTeam)
16+
[JsonConstructor]
17+
public CloudAdvocateGitHubContributorModel(IEnumerable<RepositoryPullRequest> pullRequests, string fullName, string gitHubUserName, string microsoftAlias, string microsoftTeam)
1618
: base(fullName, gitHubUserName, microsoftAlias, microsoftTeam)
1719
{
18-
PullRequests = pullReuests.ToList();
20+
PullRequests = pullRequests.ToList(); ;
1921
}
2022

23+
[JsonProperty("pullRequests")]
2124
public IReadOnlyList<RepositoryPullRequest> PullRequests { get; }
2225
}
2326
}

‎AzureAdvocates.Functions/Services/BlobStorageService.cs‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
using System;
2+
using System.Linq;
3+
using System.Threading.Tasks;
4+
using System.Collections.Generic;
5+
using Microsoft.WindowsAzure.Storage.Blob;
6+
using Newtonsoft.Json;
7+
8+
namespace AzureAdvocates.Functions
9+
{
10+
class BlobStorageService
11+
{
12+
const string _microsoftLearnContributionsContainerName = "cloudadvocatemicrosoftlearncontributions";
13+
readonly CloudBlobClient _blobClient;
14+
15+
public BlobStorageService(CloudBlobClient cloudBlobClient) => _blobClient = cloudBlobClient;
16+
17+
public Task UploadCloudAdvocateMicrosoftLearnContributions(IEnumerable<CloudAdvocateGitHubContributorModel> azureDataCenterIpRangeModel, string blobName)
18+
{
19+
var container = GetBlobContainer(_microsoftLearnContributionsContainerName);
20+
var blob = container.GetBlockBlobReference(blobName);
21+
22+
return blob.UploadTextAsync(JsonConvert.SerializeObject(azureDataCenterIpRangeModel));
23+
}
24+
25+
public async Task<IReadOnlyList<CloudAdvocateGitHubContributorModel>> GetCloudAdvocateMicrosoftLearnContributors()
26+
{
27+
var blobList = new List<CloudBlockBlob>();
28+
await foreach (var blob in GetBlobs<CloudBlockBlob>(_microsoftLearnContributionsContainerName).ConfigureAwait(false))
29+
{
30+
blobList.Add(blob);
31+
}
32+
33+
var gitHubContributorListBlob = blobList.OrderByDescending(x => x.Properties.Created).First();
34+
var serializedGitHubContributorList = await gitHubContributorListBlob.DownloadTextAsync().ConfigureAwait(false);
35+
36+
return JsonConvert.DeserializeObject<IReadOnlyList<CloudAdvocateGitHubContributorModel>>(serializedGitHubContributorList) ?? throw new NullReferenceException();
37+
}
38+
39+
async IAsyncEnumerable<T> GetBlobs<T>(string containerName, string prefix = "", int? maxresultsPerQuery = null, BlobListingDetails blobListingDetails = BlobListingDetails.None) where T : ICloudBlob
40+
{
41+
var blobContainer = GetBlobContainer(containerName);
42+
43+
BlobContinuationToken? continuationToken = null;
44+
45+
do
46+
{
47+
var response = await blobContainer.ListBlobsSegmentedAsync(prefix, true, blobListingDetails, maxresultsPerQuery, continuationToken, null, null).ConfigureAwait(false);
48+
continuationToken = response?.ContinuationToken;
49+
50+
var blobListFromResponse = response?.Results?.OfType<T>() ?? Enumerable.Empty<T>();
51+
52+
foreach (var blob in blobListFromResponse)
53+
{
54+
yield return blob;
55+
}
56+
57+
} while (continuationToken != null);
58+
59+
}
60+
61+
CloudBlobContainer GetBlobContainer(string containerName) => _blobClient.GetContainerReference(containerName);
62+
}
63+
}

‎AzureAdvocates.Functions/Startup.cs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,24 @@
22
using AzureAdvocates.Functions;
33
using GitHubReadmeWebTrends.Common;
44
using Microsoft.Azure.Functions.Extensions.DependencyInjection;
5+
using Microsoft.Extensions.DependencyInjection;
6+
using Microsoft.WindowsAzure.Storage;
7+
using Microsoft.WindowsAzure.Storage.Blob;
58

69
[assembly: FunctionsStartup(typeof(Startup))]
710
namespace AzureAdvocates.Functions
811
{
912
class Startup : FunctionsStartup
1013
{
1114
readonly static string _token = Environment.GetEnvironmentVariable("Token") ?? string.Empty;
15+
static readonly string _storageConnectionString = Environment.GetEnvironmentVariable("AzureWebJobsStorage") ?? string.Empty;
1216

13-
public override void Configure(IFunctionsHostBuilder builder) => StartupService.ConfigureServices(builder.Services, _token);
17+
public override void Configure(IFunctionsHostBuilder builder)
18+
{
19+
builder.Services.AddSingleton<CloudBlobClient>(CloudStorageAccount.Parse(_storageConnectionString).CreateCloudBlobClient());
20+
builder.Services.AddSingleton<BlobStorageService>();
21+
22+
StartupService.ConfigureServices(builder.Services, _token);
23+
}
1424
}
1525
}

‎GitHubReadmeWebTrends.Common/Models/CloudAdvocateGitHubUserModel.cs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
namespace GitHubReadmeWebTrends.Common
1+
using Newtonsoft.Json;
2+
3+
namespace GitHubReadmeWebTrends.Common
24
{
35
public class CloudAdvocateGitHubUserModel
46
{
@@ -10,9 +12,16 @@ public CloudAdvocateGitHubUserModel(in string fullName, in string gitHubUserName
1012
MicrosoftTeam = microsoftTeam;
1113
}
1214

15+
[JsonProperty("fullName")]
1316
public string FullName { get; }
17+
18+
[JsonProperty("gitHubUserName")]
1419
public string GitHubUserName { get; }
20+
21+
[JsonProperty("microsoftAlias")]
1522
public string MicrosoftAlias { get; }
23+
24+
[JsonProperty("microsoftTeam")]
1625
public string MicrosoftTeam { get; }
1726
}
1827
}

‎GitHubReadmeWebTrends.Common/Models/PullRequest.cs‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
using System;
2+
using Newtonsoft.Json;
3+
24
namespace GitHubReadmeWebTrends.Common
35
{
46
public class PullRequest
57
{
6-
[Newtonsoft.Json.JsonConstructor]
8+
[JsonConstructor]
79
public PullRequest(string id, Uri url, DateTimeOffset createdAt, bool merged, DateTimeOffset? mergedAt, string baseRefName, Author author)
810
: this(id, url, createdAt, merged, mergedAt, baseRefName, author?.Login ?? string.Empty)
911
{
@@ -21,25 +23,48 @@ public PullRequest(string id, Uri uri, DateTimeOffset createdAt, bool merged, Da
2123
Author = author;
2224
}
2325

26+
[JsonProperty("id")]
2427
public string Id { get; }
28+
29+
[JsonProperty("url")]
2530
public Uri Uri { get; }
31+
32+
[JsonProperty("createdAt")]
2633
public DateTimeOffset CreatedAt { get; }
34+
35+
[JsonProperty("merged")]
2736
public bool IsMerged { get; }
37+
38+
[JsonProperty("mergedAt")]
2839
public DateTimeOffset? MergedAt { get; }
40+
41+
[JsonProperty("baseRefName")]
2942
public string BaseRefName { get; }
43+
44+
[JsonProperty("author")]
3045
public string Author { get; }
3146
}
3247

3348
public class RepositoryPullRequest : PullRequest
3449
{
3550
public RepositoryPullRequest(string repositoryName, string repositoryOwner, PullRequest pullRequest)
36-
: base(pullRequest.Id, pullRequest.Uri, pullRequest.CreatedAt, pullRequest.IsMerged, pullRequest.MergedAt, pullRequest.BaseRefName, pullRequest.Author)
51+
: this(repositoryName, repositoryOwner, pullRequest.Id, pullRequest.Uri, pullRequest.CreatedAt, pullRequest.IsMerged, pullRequest.MergedAt, pullRequest.BaseRefName, pullRequest.Author)
52+
{
53+
54+
}
55+
56+
[JsonConstructor]
57+
public RepositoryPullRequest(string repositoryName, string repositoryOwner, string id, Uri url, DateTimeOffset createdAt, bool merged, DateTimeOffset? mergedAt, string baseRefName, string author)
58+
: base(id, url, createdAt, merged, mergedAt, baseRefName, author)
3759
{
3860
RepositoryName = repositoryName;
3961
RepositoryOwner = repositoryOwner;
4062
}
4163

64+
[JsonProperty("repositoryName")]
4265
public string RepositoryName { get; }
66+
67+
[JsonProperty("repositoryOwner")]
4368
public string RepositoryOwner { get; }
4469
}
4570
}

0 commit comments

Comments
 (0)