Skip to content

Commit

Permalink
Users/v kivlev/backport redaction to 3.227.2 (#4467)
Browse files Browse the repository at this point in the history 
* Restore expanded redaction as a default feature. Add a set of new, highly accurate detections.

* Simplify provider name.

* Add feature flag support

* Cleanup

* Set off by default / remove knob, add tracing

* added extension / fixed tests

* remove duplicate masker, changes related to tests

* Update tests, remove extra blank lines

* update agent version

---------

Co-authored-by: Michael C. Fanning <[email protected]>
Co-authored-by: v-kivlev <undefined>
  • Loading branch information
@kirill-ivlev @michaelcfanning
kirill-ivlev and michaelcfanning authored Oct 12, 2023
1 parent ba93fcd commit 9667e03
Show file tree
Hide file tree
Showing 10 changed files with 110 additions and 188 deletions.
2 changes: 1 addition & 1 deletion src/Agent.Listener/Configuration/FeatureFlagProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public async Task<FeatureFlag> GetFeatureFlagAsync(IHostContext context, string
var client = vssConnection.GetClient<FeatureAvailabilityHttpClient>();
try
{
return await client.GetFeatureFlagByNameAsync(featureFlagName);
return await client.GetFeatureFlagByNameAsync(featureFlagName, checkFeatureExists: false);
} catch(VssServiceException e)
{
Trace.Warning("Unable to retrive feature flag status: " + e.ToString());
Expand Down
14 changes: 13 additions & 1 deletion src/Agent.Listener/JobDispatcher.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
using System.Linq;
using Microsoft.VisualStudio.Services.Common;
using System.Diagnostics;

using Agent.Listener.Configuration;

namespace Microsoft.VisualStudio.Services.Agent.Listener
{
Expand Down Expand Up @@ -88,6 +88,18 @@ public void Run(Pipelines.AgentJobRequestMessage jobRequestMessage, bool runOnce
}
}

var service = HostContext.GetService<IFeatureFlagProvider>();
string ffState;
try
{
ffState = service.GetFeatureFlagAsync(HostContext, "DistributedTask.Agent.EnableAdditionalMaskingRegexes", Trace)?.Result?.EffectiveState;
}
catch (Exception)
{
ffState = "Off";
}
jobRequestMessage.Variables[Constants.Variables.Features.EnableAdditionalMaskingRegexes] = ffState;

WorkerDispatcher newDispatch = new WorkerDispatcher(jobRequestMessage.JobId, jobRequestMessage.RequestId);
if (runOnce)
{
Expand Down
6 changes: 0 additions & 6 deletions src/Agent.Sdk/Knob/AgentKnobs.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,12 +284,6 @@ public class AgentKnobs
new EnvironmentKnobSource("SYSTEM_UNSAFEALLOWMULTILINESECRET"),
new BuiltInDefaultKnobSource("false"));

public static readonly Knob MaskUsingCredScanRegexes = new Knob(
nameof(MaskUsingCredScanRegexes),
"Use the CredScan regexes for masking secrets. CredScan is an internal tool developed at Microsoft to keep passwords and authentication keys from being checked in. This defaults to disabled, as there are performance problems with some task outputs.",
new EnvironmentKnobSource("AZP_USE_CREDSCAN_REGEXES"),
new BuiltInDefaultKnobSource("false"));

public static readonly Knob MaskedSecretMinLength = new Knob(
nameof(MaskedSecretMinLength),
"Specify the length of the secrets, which, if shorter, will be ignored in the logs.",
Expand Down
14 changes: 14 additions & 0 deletions src/Agent.Worker/Worker.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using System.Threading.Tasks;
using Microsoft.VisualStudio.Services.WebApi;
using Agent.Sdk.Util;
using Agent.Sdk.Knob;

namespace Microsoft.VisualStudio.Services.Agent.Worker
{
Expand Down Expand Up @@ -67,6 +68,19 @@ public async Task<int> RunAsync(string pipeIn, string pipeOut)
InitializeSecretMasker(jobMessage);
SetCulture(jobMessage);

var maskUsingCredScanRegexesState = "Off";

if(jobMessage.Variables.TryGetValue(Constants.Variables.Agent.EnableAdditionalMaskingRegexes, out var enableAdditionalMaskingRegexes))
{
maskUsingCredScanRegexesState = enableAdditionalMaskingRegexes.Value;
}

if(maskUsingCredScanRegexesState == "On")
{
Trace.Verbose($"{Constants.Variables.Agent.EnableAdditionalMaskingRegexes} is On, adding additional masking regexes");
HostContext.AddAdditionalMaskingRegexes();
}

// Start the job.
Trace.Info($"Job message:{Environment.NewLine} {StringUtil.ConvertToJson(WorkerUtilities.ScrubPiiData(jobMessage))}");
Task<TaskResult> jobRunnerTask = jobRunner.RunAsync(jobMessage, jobRequestCancellationToken.Token);
Expand Down
204 changes: 39 additions & 165 deletions src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.CredScan.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,176 +30,50 @@ public static partial class AdditionalMaskingRegexes
private static IEnumerable<string> credScanPatterns =
new List<string>()
{
// AnsibleVaultData
@"" // pre-match
+ @"\$ANSIBLE_VAULT;\d+\.\d+;AES256\s+\d+" // match
// AAD client app, most recent two versions.
@"\b" // pre-match
+ @"[0-9A-Za-z-_~.]{3}7Q~[0-9A-Za-z-_~.]{31}\b|\b[0-9A-Za-z-_~.]{3}8Q~[0-9A-Za-z-_~.]{34}" // match
+ @"\b", // post-match

// Prominent Azure provider 512-bit symmetric keys.
@"\b" // pre-match
+ @"[0-9A-Za-z+/]{76}(APIM|ACDb|\+(ABa|AMC|ASt))[0-9A-Za-z+/]{5}[AQgw]==" // match
+ @"", // post-match

// AzurePowerShellTokenCache
@"" // pre-match
+ @"[""']TokenCache[""']\s*:\s*\{\s*[""']CacheData[""']\s*:\s*[""'][a-z0-9/\+]{86}" // match
+ @"", // post-match

// Base64EncodedStringLiteral
@"(?<=(^|[""'>;=\s#]))" // pre-match
+ @"(?<DataBlock>(?-i)MI(?i)[a-z0-9/+\s\u0085\u200b""',\\]{200,20000}[a-z0-9/+]={0,2})" // match
//
// Prominent Azure provider 256-bit symmetric keys.
@"\b" // pre-match
+ @"[0-9A-Za-z+/]{33}(AIoT|\+(ASb|AEh|ARm))[A-P][0-9A-Za-z+/]{5}=" // match
+ @"", // post-match

// JsonWebToken
@"" // pre-match
+ @"(?-i)(?<JwtToken>eyJ(?i)[a-z0-9\-_%]+\.(?-i)eyJ(?i)[a-z0-9\-_%]+\.[a-z0-9\-_%]+)|([rR]efresh_?[tT]oken|REFRESH_?TOKEN)[""']?\s*[:=]{1,2}\s*[""']?(?<JwtToken>(\w+-)+\w+)[""']?" // match
+ @"", // post-match

// SlackTokens
@"" // pre-match
+ @"xox[pbar]\-[a-z0-9\-]+" // match
+ @"", // post-match

// SymmetricKey128
@"(?<=[^\w/\+\._\$,\\])" // pre-match
+ @"(?<SymmetricKey>[a-z0-9/\+]{22}==)" // match
+ @"(?=([^\w/\+\.\$]|$))", // post-match

// SymmetricKey128Hex
@"(?<=[^\w/\+\._\$,\\][dapi]+)" // pre-match
+ @"(?<SymmetricKey>[a-f0-9]{32})" // match
+ @"(?=([^\w/\+\.\$]|$))", // post-match

// SymmetricKey160Hex
@"(?<=[^\w/\+\._\$,\\])" // pre-match
+ @"(?<Hex160>[a-f0-9/\+]{40})" // match
+ @"(?=([^\w/\+\.\$]|$))", // post-match

// SymmetricKey232
@"(?<=[^\w/\+\._\$,\\])" // pre-match
+ @"(?<SymmetricKey>(?-i)AIza(?i)[a-z0-9_\\\-]{35})" // match
+ @"(?=([^\w/\+\.\$]|$))", // post-match

// SymmetricKey240
@"(?<=[^\w/\+\.\-\$,\\])" // pre-match
+ @"(?<SymmetricKey>[a-z0-9/\+]{40})" // match
+ @"(?=([^\w/\+\.\-\$,\\]|$))", // post-match

// SymmetricKey256
@"(?<=[^\w/\+\.\$,\\])" // pre-match
+ @"(?<SymmetricKey>[a-z0-9/\+]{43}=)" // match
+ @"(?=([^\w/\+\.\$]|$))", // post-match

// SymmetricKey256B32
@"(?<=[^\w/\+\._\-\$,\\])" // pre-match
+ @"(?<SymmetricKey>(?-i)[a-z2-7]{52}(?i))" // match
+ @"(?=(?<=[0-9]+[a-z]+[0-9]+.{0,49})([^\w/\+\.\-\$,]|$))", // post-match

// SymmetricKey256UrlEncoded
@"(?<=[^\w/\+\._\-\$,\\%])" // pre-match
+ @"(?<SymmetricKey>[a-z0-9%]{43,63}%3d)" // match

// Azure Function key.
@"\b" // pre-match
+ @"[0-9A-Za-z_\-]{44}AzFu[0-9A-Za-z\-_]{5}[AQgw]==" // match
+ @"", // post-match

// SymmetricKey320
@"(?<=[^\w/\+\.\-\$,\\])" // pre-match
+ @"(?<SymmetricKey>[a-z0-9/\+]{54}={2})" // match
+ @"(?=([^\w/\+\.\-\$,\\]|$))", // post-match

// SymmetricKey320UrlEncoded
@"(?<=[^\w/\+\.\-\$,\\%])" // pre-match
+ @"(?<SymmetricKey>[a-z0-9%]{54,74}(%3d){2})" // match
// Azure Search keys.
@"\b" // pre-match
+ @"[0-9A-Za-z]{42}AzSe[A-D][0-9A-Za-z]{5}" // match
+ @"\b", // post-match

// Azure Container Registry keys.
@"\b" // pre-match
+ @"[0-9A-Za-z+/]{42}\+ACR[A-D][0-9A-Za-z+/]{5}" // match
+ @"\b", // post-match

// Azure Cache for Redis keys.
@"\b" // pre-match
+ @"[0-9A-Za-z]{33}AzCa[A-P][0-9A-Za-z]{5}=" // match
+ @"", // post-match

// SymmetricKey360
@"(?<=[^\w/\+\.\-\$,\\])" // pre-match
+ @"(?<SymmetricKey>[a-z0-9/\+]{60})" // match
+ @"(?=[^\w/\+\.\-\$,\\])", // post-match

// SymmetricKey512
@"(?<=[^\r\n\t\w/\+\.\-\$,\\])" // pre-match
+ @"(?<SymmetricKey>[a-z0-9/\+]{86}==)" // match
+ @"(?=([^\w/\+\.\-\$]|$))", // post-match

// AzureActiveDirectoryLoginCredentials
@"(?<=@([a-z0-9.]+\.(on)?)?microsoft\.com[ -~\s\u200b]{1,80}?(userpass|password|pwd|pw|\wpass[ =:>]+|(get|make)securestring)\W)" // pre-match
+ @"(?<Password>[^\s;`,""'\(\)]{10,80})" // match
+ @"(?=[\s;`,""'\(\)])", // post-match

// AzureActiveDirectoryLoginCredentials
@"" // pre-match
+ @"(?<MigrationPassword>(\-destinationPasswordPlain ""[^""]+?""))" // match
+ @"(?=[ -~\s\u200b]{1,150}?@([a-z0-9.]+\.(on)?)?microsoft\.com)", // post-match

// AzureActiveDirectoryLoginCredentials
@"(?<=(sign_in|SharePointOnlineAuthenticatedContext|(User|Exchange)Credentials?|password)[ -~\s\u200b]{1,100}?@([a-z0-9.]+\.(on)?)?microsoft\.com['""]?( \|\| \w+)?\s*,[\s\u200b]['""]?)" // pre-match
+ @"(?<ArgumentPassword>[^`'""\s,;\(\)]+?)" // match
+ @"(?=[`'""\s,;\(\)])", // post-match

// AzureActiveDirectoryLoginCredentials
@"" // pre-match
+ @"(?<PrevPassword>password[\W_][ -~\s\u200b]{40,100}?)" // match
+ @"(?=@([a-z0-9\.]+\.(on)?)?microsoft\.com)", // post-match

// LoginCredentials
@"" // pre-match
+ @"[^a-z\$](DB_USER|user id|uid|(sql)?user(name)?|service\s?account)\s*[^\w\s,]([ -~\s\u200b]{2,120}?|[ -~]{2,30}?)([^a-z\s\$]|\s)\s*(DB_PASS|(sql|service)?password|pwd)\s*[^a-z,\+&\)\]\}\[\{_][ -~\s\u200b]{2,700}?([;|<,})]|$)|[^a-z\s\$]\s*(DB_PASS|password|pwd)\s*[^a-z,\+&\)\]\}\[\{_][ -~\s\u200b]{2,60}?[^a-z\$](DB_USER|user id|uid|user(name)?)\s*[^\w\s,]([ -~\s\u200b]{2,60}?|[ -~]{2,30}?)([;|<,})]|$)" // match
+ @"", // post-match

// LoginCredentialsInUrl
@"(?<=(amqp|ssh|(ht|f)tps?)://[^%:\s""'/][^:\s""'/\$]+[^:\s""'/\$%]:)" // pre-match
+ @"(?<Password>[^%\s""'/][^@\s""'/]{0,100}[^%\s""'/])" // match
+ @"(?=@[\$a-z0-9:\.\-_%\?=/]+)", // post-match

// CertificatePrivateKeyHeader
@"" // pre-match
+ @"(?-i)\-{5}BEGIN( ([DR]SA|EC|OPENSSH|PGP))? PRIVATE KEY( BLOCK)?\-{5}" // match
+ @"", // post-match

// HttpAuthorizationHeader
@"(?<=authorization[,\[:= ""']+(basic|digest|hoba|mutual|negotiate|oauth( oauth_token=)?|bearer [^e""'&]|scram\-sha\-1|scram\-sha\-256|vapid|aws4\-hmac\-sha256)[\s\r\n]{0,10})" // pre-match
+ @"(?<Token>[a-z0-9/+_.=]{10,})" // match
+ @"", // post-match

// ClientSecretContext
@"(?<=(^|[a-z\s""'_])((app(lication)?|client)[_\- ]?(key(url)?|secret)|refresh[_\-]?token|[^t]AccessToken(Secret)?|(Consumer|api)[_\- ]?(Secret|Key)|(Twilio(Account|Auth)[_\- ]?(Sid|Token)))([\s=:>]{1,10}|[\s""':=|>,]{3,15}|[""'=:\(]{2}))" // pre-match
+ @"(?<ClientSecret>(""data:text/plain,.+""|[a-z0-9/+=_.-]{10,200}[^\(\[\{;,\r\n]|[^\s""';<,\)]{5,200}))" // match
+ @"", // post-match

// CommunityStringContext
@"(?<=(^|\W{2}|set )snmp(\-server)?( | [ -~]+? )(community|priv)\s[""']?)" // pre-match
+ @"(?<CommunityString>[^\s]+)" // match
+ @"(?=[""']?(\s|$))", // post-match

// PasswordContextInScript
@"(?<=\s-(admin|user|vm)?password\s+[""']?)" // pre-match
+ @"(?<ScriptArgumentPassword>[^$\(\[<\{\-\s,""']+)[""']?(\s|$)" // match
+ @"", // post-match

// PasswordContextInScript
@"(?<=certutil(\.exe)?.{1,10}\-p\s+[""']?)" // pre-match
+ @"(?<CertUtilPassword>[^\s,]{2,50})" // match
+ @"(?=[""']?)", // post-match

// PasswordContextInScript
@"(?<=(^|[_\s\$])[a-z]*(password|secret(key)?)[ \t]*[=:]+[ \t]*)" // pre-match
+ @"(?<ScriptAssignmentPassword>[^:\s""';,<]{2,200})" // match
+ @"", // post-match

// PasswordContextInScript
@"(?<=\s-Name\s+[""']\w+Password[""']\s+-Value\s+[""']?)" // pre-match
+ @"(?<RegistryPassword>[^\s""']{2,1100})" // match
+ @"(?=[""']?)", // post-match

// PasswordContextInScript
@"(?<=(^|[\s\r\n\\])net(\.exe)?[""'\s\\]{1,5}(user\s+|share\s+/user:)[^\s,/]+[ \t]+[""']?)" // pre-match
+ @"(?<NetUsePassword>[^\s,""'>/]{2,50})" // match
+ @"(?=[""']?)", // post-match

// PasswordContextInScript
@"(?<=psexec(\.exe)?.{1,50}-u.{1,50}-p\s+[""']?)" // pre-match
+ @"(?<PsExecPassword>[^\s,]{2,50})" // match
+ @"(?=[""']?)", // post-match

// SymmetricKeyContextInXml
@"" // pre-match
+ @"<(machineKey|parameter name=""|[a-z]+AccountInfo[^a-z])" // match
+ @"", // post-match


// NuGet API keys.
@"\b" // pre-match
+ @"oy2[a-p][0-9a-z]{15}[aq][0-9a-z]{11}[eu][bdfhjlnprtvxz357][a-p][0-9a-z]{11}[aeimquy4]" // match
+ @"\b", // post-match

// NPM author keys.
@"\b" // pre-match
+ @"npm_[0-9A-Za-z]{36}" // match
+ @"\b", // post-match
};
}
}
3 changes: 3 additions & 0 deletions src/Microsoft.VisualStudio.Services.Agent/Constants.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,7 @@ public static class Agent
public static readonly string Version = "agent.version";
public static readonly string WorkFolder = "agent.workfolder";
public static readonly string WorkingDirectory = "agent.WorkingDirectory";
public static readonly string EnableAdditionalMaskingRegexes = "agent.enableadditionalmaskingregexes";
}

public static class Build
Expand Down Expand Up @@ -371,6 +372,7 @@ public static class Features
public static readonly string GitLfsSupport = "agent.source.git.lfs";
public static readonly string GitShallowDepth = "agent.source.git.shallowFetchDepth";
public static readonly string SkipSyncSource = "agent.source.skip";
public static readonly string EnableAdditionalMaskingRegexes = "agent.enableadditionalmaskingregexes";
}

public static class Maintenance
Expand Down Expand Up @@ -511,6 +513,7 @@ public static class Task
Agent.Version,
Agent.WorkFolder,
Agent.WorkingDirectory,
Agent.EnableAdditionalMaskingRegexes,
// Build variables
Build.ArtifactStagingDirectory,
Build.BinariesDirectory,
Expand Down
18 changes: 11 additions & 7 deletions src/Microsoft.VisualStudio.Services.Agent/HostContext.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
using System.Net.Http.Headers;
using Pipelines = Microsoft.TeamFoundation.DistributedTask.Pipelines;
using Agent.Sdk.Util;
using BuildXL.Cache.ContentStore.Interfaces.Tracing;

namespace Microsoft.VisualStudio.Services.Agent
{
Expand Down Expand Up @@ -88,6 +89,7 @@ public class HostContext : EventListener, IObserver<DiagnosticListener>, IObserv
public ShutdownReason AgentShutdownReason { get; private set; }
public ILoggedSecretMasker SecretMasker => _secretMasker;
public ProductInfoHeaderValue UserAgent => _userAgent;

public HostContext(HostType hostType, string logFile = null)
{
_secretMasker = new LoggedSecretMasker(_basicSecretMasker);
Expand All @@ -106,13 +108,6 @@ public HostContext(HostType hostType, string logFile = null)
this.SecretMasker.AddValueEncoder(ValueEncoders.UriDataEscape, $"HostContext_{WellKnownSecretAliases.UriDataEscape}");
this.SecretMasker.AddValueEncoder(ValueEncoders.BackslashEscape, $"HostContext_{WellKnownSecretAliases.UriDataEscape}");
this.SecretMasker.AddRegex(AdditionalMaskingRegexes.UrlSecretPattern, $"HostContext_{WellKnownSecretAliases.UrlSecretPattern}");
if (AgentKnobs.MaskUsingCredScanRegexes.GetValue(this).AsBoolean())
{
foreach (var pattern in AdditionalMaskingRegexes.CredScanPatterns)
{
this.SecretMasker.AddRegex(pattern, $"HostContext_{WellKnownSecretAliases.CredScanPatterns}");
}
}

// Create the trace manager.
if (string.IsNullOrEmpty(logFile))
Expand Down Expand Up @@ -741,6 +736,15 @@ public static HttpClientHandler CreateHttpClientHandler(this IHostContext contex

return clientHandler;
}

public static void AddAdditionalMaskingRegexes(this IHostContext context)
{
ArgUtil.NotNull(context, nameof(context));
foreach (var pattern in AdditionalMaskingRegexes.CredScanPatterns)
{
context.SecretMasker.AddRegex(pattern, $"HostContext_{WellKnownSecretAliases.CredScanPatterns}");
}
}
}

public enum ShutdownReason
Expand Down
Loading

0 comments on commit 9667e03

Please sign in to comment.