using System;
using System.Collections.Generic;
using System.Text;
namespace SyncFromLocalToS3
{
class Program
{
private static SprightlySoftAWS.S3.UploadEstimate MyUpload;
private static Boolean ShowUploadProgress;
private static int ExitCode;
private static String EmailLogData;
private static int LogLevel;
private static String LogFilePath;
private static String SoureFolder;
private static int EmailLogCondition;
private static String EmailSmtpHost;
private static int EmailSmtpPort;
private static Boolean EmailSmtpEnableSsl;
private static String EmailSmtpUserName;
private static String EmailSmtpPassword;
private static String EmailFromAddress;
private static String EmailRecipients;
static void Main(string[] args)
{
AppDomain currentDomain = AppDomain.CurrentDomain;
currentDomain.UnhandledException += new UnhandledExceptionEventHandler(MyHandler);
ExitCode = 0;
EmailLogData = "";
String S3FolderDelimiter = "/";
Boolean LogOnlyMode = Convert.ToBoolean(GetValueFromConfigOrArgument(args, "LogOnlyMode"));
LogLevel = Convert.ToInt32(GetValueFromConfigOrArgument(args, "LogLevel"));
LogFilePath = GetValueFromConfigOrArgument(args, "LogFilePath");
Boolean OverwriteExistingLogFile = Convert.ToBoolean(GetValueFromConfigOrArgument(args, "OverwriteExistingLogFile"));
String AWSAccessKeyId = GetValueFromConfigOrArgument(args, "AWSAccessKeyId");
String AWSSecretAccessKey = GetValueFromConfigOrArgument(args, "AWSSecretAccessKey");
Boolean UseSSL = Convert.ToBoolean(GetValueFromConfigOrArgument(args, "UseSSL"));
String RequestEndpoint = GetValueFromConfigOrArgument(args, "RequestEndpoint");
String BucketName = GetValueFromConfigOrArgument(args, "BucketName");
String UploadPrefix = GetValueFromConfigOrArgument(args, "UploadPrefix");
String UploadHeaders = GetValueFromConfigOrArgument(args, "UploadHeaders");
String OverrideContentTypes = GetValueFromConfigOrArgument(args, "OverrideContentTypes");
Boolean CalculateMD5ForUpload = Convert.ToBoolean(GetValueFromConfigOrArgument(args, "CalculateMD5ForUpload"));
Boolean SaveTimestampsInMetadata = Convert.ToBoolean(GetValueFromConfigOrArgument(args, "SaveTimestampsInMetadata"));
Single UploadSpeedLimitKBps = Convert.ToSingle(GetValueFromConfigOrArgument(args, "UploadSpeedLimitKBps"));
ShowUploadProgress = Convert.ToBoolean(GetValueFromConfigOrArgument(args, "ShowUploadProgress"));
int S3ErrorRetries = Convert.ToInt32(GetValueFromConfigOrArgument(args, "S3ErrorRetries"));
SoureFolder = GetValueFromConfigOrArgument(args, "SoureFolder");
Boolean IncludeSubFolders = Convert.ToBoolean(GetValueFromConfigOrArgument(args, "IncludeSubFolders"));
String ExcludeFolders = GetValueFromConfigOrArgument(args, "ExcludeFolders");
String IncludeOnlyFilesRegularExpression = GetValueFromConfigOrArgument(args, "IncludeOnlyFilesRegularExpression");
String ExcludeFilesRegularExpression = GetValueFromConfigOrArgument(args, "ExcludeFilesRegularExpression");
String CompareFilesBy = GetValueFromConfigOrArgument(args, "CompareFilesBy");
Boolean DeleteS3ItemsWhereNotInLocalList = Convert.ToBoolean(GetValueFromConfigOrArgument(args, "DeleteS3ItemsWhereNotInLocalList"));
EmailLogCondition = Convert.ToInt32(GetValueFromConfigOrArgument(args, "EmailLogCondition"));
EmailSmtpHost = GetValueFromConfigOrArgument(args, "EmailSmtpHost");
EmailSmtpPort = Convert.ToInt32(GetValueFromConfigOrArgument(args, "EmailSmtpPort"));
EmailSmtpEnableSsl = Convert.ToBoolean(GetValueFromConfigOrArgument(args, "EmailSmtpEnableSsl"));
EmailSmtpUserName = GetValueFromConfigOrArgument(args, "EmailSmtpUserName");
EmailSmtpPassword = GetValueFromConfigOrArgument(args, "EmailSmtpPassword");
EmailFromAddress = GetValueFromConfigOrArgument(args, "EmailFromAddress");
EmailRecipients = GetValueFromConfigOrArgument(args, "EmailRecipients");
//Initialize the log file.
InitializeLogFile(OverwriteExistingLogFile);
WriteToLog("");
WriteToLog("Program started. " + DateTime.Now);
WriteToLog("", 2);
WriteToLog("Input parameters:", 2);
WriteToLog(" AWSAccessKeyId=" + AWSAccessKeyId, 2);
WriteToLog(" AWSSecretAccessKey=" + AWSSecretAccessKey, 2);
WriteToLog(" UseSSL=" + UseSSL, 2);
WriteToLog(" RequestEndpoint=" + RequestEndpoint, 2);
WriteToLog(" BucketName=" + BucketName, 2);
WriteToLog(" UploadPrefix=" + UploadPrefix, 2);
WriteToLog(" OverrideContentTypes=" + OverrideContentTypes, 2);
WriteToLog(" CalculateMD5ForUpload=" + CalculateMD5ForUpload, 2);
WriteToLog(" SaveTimestampsInMetadata=" + SaveTimestampsInMetadata, 2);
WriteToLog(" UploadSpeedLimitKBps=" + UploadSpeedLimitKBps, 2);
WriteToLog(" ShowUploadProgress=" + ShowUploadProgress, 2);
WriteToLog(" S3ErrorRetries=" + S3ErrorRetries, 2);
WriteToLog(" SoureFolder=" + SoureFolder, 2);
WriteToLog(" IncludeSubFolders=" + IncludeSubFolders, 2);
WriteToLog(" ExcludeFolders=" + ExcludeFolders, 2);
WriteToLog(" IncludeOnlyFilesRegularExpression=" + IncludeOnlyFilesRegularExpression, 2);
WriteToLog(" ExcludeFilesRegularExpression=" + ExcludeFilesRegularExpression, 2);
WriteToLog(" CompareFilesBy=" + CompareFilesBy, 2);
WriteToLog(" DeleteS3ItemsWhereNotInLocalList=" + DeleteS3ItemsWhereNotInLocalList, 2);
WriteToLog(" LogOnlyMode=" + LogOnlyMode, 2);
WriteToLog(" LogLevel=" + LogLevel, 2);
WriteToLog(" LogFilePath=" + LogFilePath, 2);
WriteToLog(" OverwriteExistingLogFile=" + OverwriteExistingLogFile, 2);
WriteToLog(" EmailLogCondition=" + EmailLogCondition, 2);
WriteToLog(" EmailSmtpHost=" + EmailSmtpHost, 2);
WriteToLog(" EmailSmtpPort=" + EmailSmtpPort, 2);
WriteToLog(" EmailSmtpEnableSsl=" + EmailSmtpEnableSsl, 2);
WriteToLog(" EmailSmtpUserName=" + EmailSmtpUserName, 2);
WriteToLog(" EmailSmtpPassword=" + EmailSmtpPassword, 2);
WriteToLog(" EmailFromAddress=" + EmailFromAddress, 2);
WriteToLog(" EmailRecipients=" + EmailRecipients, 2);
//Check that the input values are valid.
if (AWSAccessKeyId == "")
{
WriteToLog("Error, the AWSAccessKeyId is empty. Set it in the App.config file or pass it in the command line.");
ExitCode = 1;
}
else if (AWSSecretAccessKey == "")
{
WriteToLog("Error, the AWSSecretAccessKey is empty. Set it in the App.config file or pass it in the command line.");
ExitCode = 1;
}
else if (BucketName == "")
{
WriteToLog("Error, the BucketName is empty. Set it in the App.config file or pass it in the command line.");
ExitCode = 1;
}
else if (SoureFolder == "")
{
WriteToLog("Error, the SoureFolder is empty. Set it in the App.config file or pass it in the command line.");
ExitCode = 1;
}
else if (System.IO.Directory.Exists(SoureFolder) == false)
{
WriteToLog("Error, the SoureFolder does not exist. Specify a valif path. SoureFolder=" + SoureFolder);
ExitCode = 1;
}
else if (UploadPrefix != "" && UploadPrefix.EndsWith(S3FolderDelimiter) == false)
{
WriteToLog("Error, the UploadPrefix is the S3 folder to sync with. It must end with the S3FolderDelimiter value. UploadPrefix=" + UploadPrefix + " S3FolderDelimiter=" + S3FolderDelimiter);
ExitCode = 1;
}
else if (UploadPrefix != "" && GetS3FileExists(AWSAccessKeyId, AWSSecretAccessKey, UseSSL, RequestEndpoint, BucketName, UploadPrefix, S3ErrorRetries) != 200)
{
WriteToLog("Error, the UploadPrefix folder does not exist on S3. UploadPrefix=" + UploadPrefix);
ExitCode = 1;
}
if (ExitCode == 0)
{
//Make sure source path ends with the backslash character.
if (SoureFolder.EndsWith("\\") == false)
{
SoureFolder += "\\";
}
MyUpload = new SprightlySoftAWS.S3.UploadEstimate();
MyUpload.ProgressChangedEvent += MyUpload_ProgressChangedEvent;
//Convert the user's UploadHeaders into a dictionary.
Dictionary<String, String> UserRequestHeaders;
UserRequestHeaders = BuildDictionaryFromString(UploadHeaders);
//Convert the user's OverrideContentTypes into a dictionary.
Dictionary<String, String> UserContentTypes;
UserContentTypes = BuildDictionaryFromString(OverrideContentTypes);
//Build an ArrayList of the user's folders to exclude.
System.Collections.ArrayList ExcludeFolderslArrayList = new System.Collections.ArrayList();
if (ExcludeFolders != "")
{
System.IO.DirectoryInfo MyDirectoryInfo;
String[] ExcludeLocalArray = ExcludeFolders.Split(new Char[] { '|' });
foreach (String ArrayValue in ExcludeLocalArray)
{
if (System.IO.Directory.Exists(ArrayValue) == true)
{
MyDirectoryInfo = new System.IO.DirectoryInfo(ArrayValue);
ExcludeFolderslArrayList.Add(MyDirectoryInfo.FullName.ToLower());
}
}
}
System.Collections.Hashtable S3HashTable = new System.Collections.Hashtable();
System.Collections.ArrayList LocalArrayList = new System.Collections.ArrayList();
Boolean RetBool;
//List all items on S3 and put them in a HashTable.
WriteToLog("", 2);
WriteToLog("Getting files on S3.");
RetBool = PopulateS3HashTable(AWSAccessKeyId, AWSSecretAccessKey, UseSSL, RequestEndpoint, BucketName, UploadPrefix, ref S3HashTable);
WriteToLog(" Number of items in S3 list: " + S3HashTable.Count);
if (RetBool == true)
{
//List all local items to sync and put them in an ArrayList.
WriteToLog("");
WriteToLog("Getting local files.");
PopulateLocalArrayList(SoureFolder, SoureFolder, IncludeSubFolders, ExcludeFolderslArrayList, IncludeOnlyFilesRegularExpression, ExcludeFilesRegularExpression, ref LocalArrayList);
WriteToLog(" Number of items in local list: " + LocalArrayList.Count);
//Create an ArrayList of S3 items that do not exist in the LocalArrayList.
System.Collections.ArrayList DeleteS3ArrayList = new System.Collections.ArrayList();
PopulateDeleteS3ArrayList(ref S3HashTable, ref LocalArrayList, UploadPrefix, S3FolderDelimiter, SoureFolder, ref DeleteS3ArrayList);
//Use a Dictionary to store files to upload. We may be calculating the ETag hash and we can pass these values to the upload function in the dictionary.
Dictionary<String, String> UploadDictionary = new Dictionary<String, String>();
PopulateUploadDictionary(ref S3HashTable, ref LocalArrayList, UploadPrefix, S3FolderDelimiter, SoureFolder, CompareFilesBy, ref UploadDictionary);
if (LogOnlyMode == true)
{
//List what would be changed on S3.
if (DeleteS3ItemsWhereNotInLocalList == true)
{
//List items that would be deleted on S3.
WriteToLog("");
WriteToLog("The following items would be deleted on S3:");
foreach (String DeleteItem in DeleteS3ArrayList)
{
WriteToLog(" " + DeleteItem);
}
WriteToLog(" Number of items that would be deleted: " + DeleteS3ArrayList.Count);
}
//List items that would be uploaded to S3.
WriteToLog("");
WriteToLog("The following items would be uploaded to S3:");
foreach (KeyValuePair<String, String> UploadKeyValuePair in UploadDictionary)
{
WriteToLog(" " + UploadKeyValuePair.Key);
}
WriteToLog(" Number of items that would be uploaded: " + UploadDictionary.Count);
}
else
{
//Make the changes on S3.
if (DeleteS3ItemsWhereNotInLocalList == true)
{
//Process delete queue.
WriteToLog("");
WriteToLog("Deleting items on S3:");
DeleteExtraOnS3(AWSAccessKeyId, AWSSecretAccessKey, UseSSL, RequestEndpoint, BucketName, ref DeleteS3ArrayList, S3ErrorRetries);
}
//Process upload queue.
WriteToLog("");
WriteToLog("Uploading items on S3:");
UploadMissingToS3(AWSAccessKeyId, AWSSecretAccessKey, UseSSL, RequestEndpoint, BucketName, UploadPrefix, S3FolderDelimiter, SoureFolder, UserRequestHeaders, UserContentTypes, CalculateMD5ForUpload, ref UploadDictionary, SaveTimestampsInMetadata, UploadSpeedLimitKBps, ShowUploadProgress, S3ErrorRetries);
}
}
}
SendEmailLog();
Environment.ExitCode = ExitCode;
WriteToLog("");
WriteToLog("Program complete. " + DateTime.Now);
}
private static Boolean PopulateS3HashTable(String AWSAccessKeyId, String AWSSecretAccessKey, Boolean UseSSL, String RequestEndpoint, String BucketName, String UploadPrefix, ref System.Collections.Hashtable S3HashTable)
{
Boolean RetBool;
SprightlySoftAWS.S3.ListBucket MyListBucket = new SprightlySoftAWS.S3.ListBucket();
RetBool = MyListBucket.ListBucket(UseSSL, RequestEndpoint, BucketName, "", UploadPrefix, AWSAccessKeyId, AWSSecretAccessKey);
if (RetBool == true)
{
foreach (SprightlySoftAWS.S3.ListBucket.BucketItemObject MyBucketItemObject in MyListBucket.BucketItemsArrayList)
{
WriteToLog(" Item added to S3 list. KeyName=" + MyBucketItemObject.KeyName, 2);
S3HashTable.Add(MyBucketItemObject.KeyName, MyBucketItemObject);
}
}
else
{
WriteToLog(" Error listing files on S3. ErrorDescription=" + MyListBucket.ErrorDescription);
WriteToLog(MyListBucket.LogData, 3);
}
return RetBool;
}
private static void PopulateLocalArrayList(String BaseFolder, String CurrentFolder, Boolean IncludeSubFolders, System.Collections.ArrayList ExcludeFolderslArrayList, String IncludeOnlyFilesRegularExpression, String ExcludeFilesRegularExpression, ref System.Collections.ArrayList LocalArrayList)
{
System.IO.DirectoryInfo CurrentDirectoryInfo;
CurrentDirectoryInfo = new System.IO.DirectoryInfo(CurrentFolder);
String FolderName;
foreach (System.IO.FileInfo MyFileInfo in CurrentDirectoryInfo.GetFiles())
{
if (ExcludeFilesRegularExpression != "")
{
//check if the file is excluded
if (System.Text.RegularExpressions.Regex.IsMatch(MyFileInfo.FullName, ExcludeFilesRegularExpression, System.Text.RegularExpressions.RegexOptions.IgnoreCase) == true)
{
WriteToLog(" Local file excluded by ExcludeFilesRegularExpression. Name=" + MyFileInfo.FullName, 2);
}
else
{
if (IncludeOnlyFilesRegularExpression != "")
{
if (System.Text.RegularExpressions.Regex.IsMatch(MyFileInfo.FullName, IncludeOnlyFilesRegularExpression, System.Text.RegularExpressions.RegexOptions.IgnoreCase) == true)
{
WriteToLog(" File added to local list. Name=" + MyFileInfo.FullName, 2);
LocalArrayList.Add(MyFileInfo.FullName);
}
else
{
WriteToLog(" Local file not included by IncludeOnlyFilesRegularExpression. Name=" + MyFileInfo.FullName, 2);
}
}
else
{
WriteToLog(" File added to local list. Name=" + MyFileInfo.FullName, 2);
LocalArrayList.Add(MyFileInfo.FullName);
}
}
}
else
{
if (IncludeOnlyFilesRegularExpression != "")
{
if (System.Text.RegularExpressions.Regex.IsMatch(MyFileInfo.FullName, IncludeOnlyFilesRegularExpression, System.Text.RegularExpressions.RegexOptions.IgnoreCase) == true)
{
WriteToLog(" File added to list. Name=" + MyFileInfo.FullName, 2);
LocalArrayList.Add(MyFileInfo.FullName);
}
else
{
WriteToLog(" Local file not included by IncludeOnlyFilesRegularExpression. Name=" + MyFileInfo.FullName, 2);
}
}
else
{
WriteToLog(" File added to local list. Name=" + MyFileInfo.FullName, 2);
LocalArrayList.Add(MyFileInfo.FullName);
}
}
}
if (IncludeSubFolders == true)
{
foreach (System.IO.DirectoryInfo SubDirectoryInfo in CurrentDirectoryInfo.GetDirectories())
{
if (ExcludeFolderslArrayList.Contains(SubDirectoryInfo.FullName.ToLower()) == true)
{
WriteToLog(" Local folder excluded by ExcludeFolders. Name=" + SubDirectoryInfo.FullName, 2);
}
else
{
FolderName = SubDirectoryInfo.FullName;
if (FolderName.EndsWith("\\") == false)
{
FolderName += "\\";
}
WriteToLog(" Folder added to local list. Name=" + FolderName, 2);
LocalArrayList.Add(FolderName);
PopulateLocalArrayList(BaseFolder, SubDirectoryInfo.FullName, IncludeSubFolders, ExcludeFolderslArrayList, IncludeOnlyFilesRegularExpression, ExcludeFilesRegularExpression, ref LocalArrayList);
}
}
}
else
{
WriteToLog(" Sub folders excluded by IncludeSubFolders.", 2);
}
}
private static void PopulateDeleteS3ArrayList(ref System.Collections.Hashtable S3HashTable, ref System.Collections.ArrayList LocalArrayList, String UploadPrefix, String S3FolderDelimiter, String SoureFolder, ref System.Collections.ArrayList DeleteS3ArrayList)
{
//For each S3 item, check if it exists locally.
String KeyName;
String LocalPath;
foreach (System.Collections.DictionaryEntry MyDictionaryEntry in S3HashTable)
{
KeyName = MyDictionaryEntry.Key.ToString();
KeyName = KeyName.Substring(UploadPrefix.Length);
LocalPath = System.IO.Path.Combine(SoureFolder, KeyName.Replace(S3FolderDelimiter, "\\"));
if (LocalArrayList.Contains(LocalPath) == false)
{
DeleteS3ArrayList.Add(MyDictionaryEntry.Key);
}
}
}
private static void PopulateUploadDictionary(ref System.Collections.Hashtable S3HashTable, ref System.Collections.ArrayList LocalArrayList, String UploadPrefix, String S3FolderDelimiter, String SoureFolder, String CompareFilesBy, ref Dictionary<String, String> UploadDictionary)
{
//Check which local items need to be uploaded to S3.
String LocalPathAsKey;
SprightlySoftAWS.S3.CalculateHash MyCalculateHash = new SprightlySoftAWS.S3.CalculateHash();
String LocalETag;
System.IO.FileInfo MyFileInfo;
CompareFilesBy = CompareFilesBy.ToLower();
foreach (String LocalPath in LocalArrayList)
{
LocalPathAsKey = LocalPath;
LocalPathAsKey = LocalPathAsKey.Substring(SoureFolder.Length);
LocalPathAsKey = LocalPathAsKey.Replace("\\", S3FolderDelimiter);
LocalPathAsKey = System.IO.Path.Combine(UploadPrefix, LocalPathAsKey);
if (S3HashTable.ContainsKey(LocalPathAsKey) == true)
{
//Only check files to see if the content is different.
if (LocalPath.EndsWith("\\") == false)
{
//The local file exists on S3. Check if the files are different.
SprightlySoftAWS.S3.ListBucket.BucketItemObject MyBucketItemObject;
MyBucketItemObject = S3HashTable[LocalPathAsKey] as SprightlySoftAWS.S3.ListBucket.BucketItemObject;
if (CompareFilesBy == "etag")
{
LocalETag = MyCalculateHash.CalculateETagFromFile(LocalPath);
if (LocalETag == MyBucketItemObject.ETag.Replace("\"", ""))
{
//Files are the same.
}
else
{
UploadDictionary.Add(LocalPath, LocalETag);
}
}
else if (CompareFilesBy == "size")
{
MyFileInfo = new System.IO.FileInfo(LocalPath);
if (MyFileInfo.Length == MyBucketItemObject.Size)
{
//Files are the same.
}
else
{
UploadDictionary.Add(LocalPath, "");
}
}
else
{
//If the FileName is different the file will not exists on S3. No need to do a check here.
}
}
}
else
{
//The local file does not exist on S3, add it to the upload list.
UploadDictionary.Add(LocalPath, "");
}
}
}
private static void DeleteExtraOnS3(String AWSAccessKeyId, String AWSSecretAccessKey, Boolean UseSSL, String RequestEndpoint, String BucketName, ref System.Collections.ArrayList DeleteS3ArrayList, int S3ErrorRetries)
{
Boolean RetBool;
int ErrorNumber = 0;
String ErrorDescription = "";
String LogData = "";
int ResponseStatusCode = 0;
String ResponseStatusDescription = "";
Dictionary<String, String> ResponseHeaders = new Dictionary<String, String>();
String ResponseString = "";
int DeleteCount = 0;
foreach (String DeleteItem in DeleteS3ArrayList)
{
//Send the correct parameters to the MakeS3Request function to delete a file on S3. This function will wait and retry if there is a 503 error.
RetBool = MakeS3Request(AWSAccessKeyId, AWSSecretAccessKey, UseSSL, RequestEndpoint, BucketName, DeleteItem, "", "DELETE", null, "", S3ErrorRetries, ref ErrorNumber, ref ErrorDescription, ref LogData, ref ResponseStatusCode, ref ResponseStatusDescription, ref ResponseHeaders, ref ResponseString);
if (RetBool == true)
{
WriteToLog(" Delete S3 file successful. S3KeyName=" + DeleteItem);
DeleteCount += 1;
}
else
{
WriteToLog(" Delete S3 file failed. S3KeyName=" + DeleteItem + " ErrorNumber=" + ErrorNumber + " ErrorDescription=" + ErrorDescription + " ResponseString=" + ResponseString);
WriteToLog(LogData, 2);
WriteToLog(" Canceling deletion of extra files on S3.");
ExitCode = 1;
break;
}
}
WriteToLog(" Number of items deleted: " + DeleteCount);
}
private static Boolean MakeS3Request(String AWSAccessKeyId, String AWSSecretAccessKey, Boolean UseSSL, String RequestEndpoint, String BucketName, String KeyName, String QueryString, String RequestMethod, Dictionary<String, String> ExtraHeaders, String SendData, int RetryTimes, ref int ErrorNumber, ref String ErrorDescription, ref String LogData, ref int ResponseStatusCode, ref String ResponseStatusDescription, ref Dictionary<String, String> ResponseHeaders, ref String ResponseString)
{
SprightlySoftAWS.REST MyREST = new SprightlySoftAWS.REST();
String RequestURL;
Dictionary<String, String> ExtraRequestHeaders;
String AuthorizationValue;
Boolean RetBool = true;
LogData = "";
for (int i = 0; i == RetryTimes; i++)
{
RequestURL = MyREST.BuildS3RequestURL(UseSSL, RequestEndpoint, BucketName, KeyName, QueryString);
ExtraRequestHeaders = new Dictionary<String, String>();
if (ExtraHeaders != null)
{
foreach (KeyValuePair<String, String> MyKeyValuePair in ExtraHeaders)
{
ExtraRequestHeaders.Add(MyKeyValuePair.Key, MyKeyValuePair.Value);
}
}
ExtraRequestHeaders.Add("x-amz-date", DateTime.UtcNow.ToString("r"));
AuthorizationValue = MyREST.GetS3AuthorizationValue(RequestURL, RequestMethod, ExtraRequestHeaders, AWSAccessKeyId, AWSSecretAccessKey);
ExtraRequestHeaders.Add("Authorization", AuthorizationValue);
RetBool = MyREST.MakeRequest(RequestURL, RequestMethod, ExtraRequestHeaders, SendData);
//Set the return values.
ErrorNumber = MyREST.ErrorNumber;
ErrorDescription = MyREST.ErrorDescription;
LogData += MyREST.LogData;
ResponseStatusCode = MyREST.ResponseStatusCode;
ResponseStatusDescription = MyREST.ResponseStatusDescription;
ResponseHeaders = MyREST.ResponseHeaders;
ResponseString = MyREST.ResponseString;
if (RetBool == true)
{
break;
}
else
{
if (MyREST.ResponseStatusCode == 503)
{
//A Service Unavailable response was returned. Wait and retry.
System.Threading.Thread.Sleep(1000 * i * i);
}
else if (MyREST.ErrorNumber == 1003)
{
//Getting the response failed. This may be a network disconnection. Wait and retry.
System.Threading.Thread.Sleep(1000 * i * i);
}
else
{
//An error occured but retrying would not solve the problem.
break;
}
}
}
return RetBool;
}
private static void UploadMissingToS3(String AWSAccessKeyId, String AWSSecretAccessKey, Boolean UseSSL, String RequestEndpoint, String BucketName, String UploadPrefix, String S3FolderDelimiter, String SoureFolder, Dictionary<String, String> UserRequestHeaders, Dictionary<String, String> UserContentTypes, Boolean CalculateMD5ForUpload, ref Dictionary<String, String> UploadDictionary, Boolean SaveTimestampsInMetadata, Single UploadSpeedLimitKBps, Boolean ShowUploadProgress, int S3ErrorRetries)
{
Boolean RetBool;
int ErrorNumber = 0;
String ErrorDescription = "";
String LogData = "";
int ResponseStatusCode = 0;
String ResponseStatusDescription = "";
Dictionary<String, String> ResponseHeaders = new Dictionary<String, String>();
String ResponseString = "";
Dictionary<String, String> ExtraHeaders = new Dictionary<String, String>();
String DiffName;
String LocalMD5Hash;
String MyExtension;
String KeyName;
System.IO.FileInfo MyFileInfo;
System.IO.DirectoryInfo MyDirectoryInfo;
int UploadCount = 0;
SprightlySoftAWS.S3.CalculateHash MyCalculateHash = new SprightlySoftAWS.S3.CalculateHash();
SprightlySoftAWS.S3.Helper MyS3Helper = new SprightlySoftAWS.S3.Helper();
//Get a dictionary of content types from the SprightlySoftAWS.S3.Helper class.
Dictionary<String, String> ContentTypesDictionary;
ContentTypesDictionary = MyS3Helper.GetContentTypesDictionary();
foreach (KeyValuePair<String, String> UploadKeyValuePair in UploadDictionary)
{
DiffName = UploadKeyValuePair.Key.Substring(SoureFolder.Length, UploadKeyValuePair.Key.Length - SoureFolder.Length);
DiffName = DiffName.Replace("\\", "/");
KeyName = UploadPrefix + DiffName;
if (UploadKeyValuePair.Key.EndsWith("\\") == true)
{
ExtraHeaders = new Dictionary<String, String>();
if (SaveTimestampsInMetadata == true)
{
MyDirectoryInfo = new System.IO.DirectoryInfo(UploadKeyValuePair.Key);
ExtraHeaders.Add("x-amz-meta-local-date-created", MyDirectoryInfo.CreationTime.ToFileTimeUtc().ToString());
}
RetBool = MakeS3Request(AWSAccessKeyId, AWSSecretAccessKey, UseSSL, RequestEndpoint, BucketName, KeyName, "", "PUT", ExtraHeaders, "", S3ErrorRetries, ref ErrorNumber, ref ErrorDescription, ref LogData, ref ResponseStatusCode, ref ResponseStatusDescription, ref ResponseHeaders, ref ResponseString);
if (RetBool == true)
{
WriteToLog(" Create S3 folder successful. S3KeyName=" + KeyName);
UploadCount += 1;
}
else
{
WriteToLog(" Create S3 folder failed. S3KeyName=" + KeyName + " ErrorNumber=" + ErrorNumber + " ErrorDescription=" + ErrorDescription + " ResponseString=" + ResponseString);
WriteToLog(LogData, 2);
WriteToLog(" Canceling upload of missing files to S3.");
ExitCode = 1;
break;
}
}
else
{
//Calculate the MD5 for upload if required.
//Set the content type.
//Add extra headers.
ExtraHeaders = new Dictionary<String, String>();
if (CalculateMD5ForUpload == true)
{
if (UploadKeyValuePair.Value == "")
{
//Calculate the MD5.
LocalMD5Hash = MyCalculateHash.CalculateMD5FromFile(UploadKeyValuePair.Key);
}
else
{
//Convert the ETag to MD5.
LocalMD5Hash = MyS3Helper.ConvertETagToMD5(UploadKeyValuePair.Value);
}
ExtraHeaders.Add("Content-MD5", LocalMD5Hash);
}
MyExtension = System.IO.Path.GetExtension(UploadKeyValuePair.Key).ToLower();
if (UserContentTypes.ContainsKey(MyExtension) == true)
{
ExtraHeaders.Add("Content-Type", UserContentTypes[MyExtension]);
}
else if (ContentTypesDictionary.ContainsKey(MyExtension) == true)
{
ExtraHeaders.Add("Content-Type", ContentTypesDictionary[MyExtension]);
}
if (SaveTimestampsInMetadata == true)
{
MyFileInfo = new System.IO.FileInfo(UploadKeyValuePair.Key);
ExtraHeaders.Add("x-amz-meta-local-date-modified", MyFileInfo.LastWriteTimeUtc.ToFileTimeUtc().ToString());
ExtraHeaders.Add("x-amz-meta-local-date-created", MyFileInfo.CreationTime.ToFileTimeUtc().ToString());
}
//Add the user supplied headers to the other headers that will be sent to S3.
foreach (KeyValuePair<String, String> MyKeyValuePair in UserRequestHeaders)
{
ExtraHeaders.Add(MyKeyValuePair.Key, MyKeyValuePair.Value);
}
WriteToLog(" Uploading file. S3KeyName=" + KeyName);
RetBool = UploadFileToS3(AWSAccessKeyId, AWSSecretAccessKey, UseSSL, RequestEndpoint, BucketName, KeyName, "PUT", ExtraHeaders, UploadKeyValuePair.Key, UploadSpeedLimitKBps, S3ErrorRetries, ref ErrorNumber, ref ErrorDescription, ref LogData, ref ResponseStatusCode, ref ResponseStatusDescription, ref ResponseHeaders, ref ResponseString);
if (RetBool == true)
{
WriteToLog(" Upload file to S3 was successful.");
UploadCount += 1;
}
else
{
WriteToLog(" Upload file to S3 failed. ErrorNumber=" + ErrorNumber + " ErrorDescription=" + ErrorDescription + " ResponseString=" + ResponseString);
WriteToLog(LogData, 2);
WriteToLog(" Canceling upload of missing files to S3.");
ExitCode = 1;
break;
}
}
}
WriteToLog(" Number of items uploaded: " + UploadCount);
}
private static Boolean UploadFileToS3(String AWSAccessKeyId, String AWSSecretAccessKey, Boolean UseSSL, String RequestEndpoint, String BucketName, String KeyName, String RequestMethod, Dictionary<String, String> ExtraHeaders, String LocalFileName, Single UploadSpeedLimitKBps, int RetryTimes, ref int ErrorNumber, ref String ErrorDescription, ref String LogData, ref int ResponseStatusCode, ref String ResponseStatusDescription, ref Dictionary<String, String> ResponseHeaders, ref String ResponseString)
{
String RequestURL;
Dictionary<String, String> ExtraRequestHeaders;
String AuthorizationValue;
Boolean RetBool = true;
LogData = "";
if (UploadSpeedLimitKBps > 0)
{
MyUpload.LimitKBpsSpeed = UploadSpeedLimitKBps;
}
for (int i = 0; i == RetryTimes; i++)
{
RequestURL = MyUpload.BuildS3RequestURL(UseSSL, RequestEndpoint, BucketName, KeyName, "");
ExtraRequestHeaders = new Dictionary<String, String>();
if (ExtraHeaders != null)
{
foreach (KeyValuePair<String, String> MyKeyValuePair in ExtraHeaders)
{
ExtraRequestHeaders.Add(MyKeyValuePair.Key, MyKeyValuePair.Value);
}
}
ExtraRequestHeaders.Add("x-amz-date", DateTime.UtcNow.ToString("r"));
AuthorizationValue = MyUpload.GetS3AuthorizationValue(RequestURL, RequestMethod, ExtraRequestHeaders, AWSAccessKeyId, AWSSecretAccessKey);
ExtraRequestHeaders.Add("Authorization", AuthorizationValue);
RetBool = MyUpload.UploadFile(RequestURL, RequestMethod, ExtraRequestHeaders, LocalFileName);
//Set the return values.
ErrorNumber = MyUpload.ErrorNumber;
ErrorDescription = MyUpload.ErrorDescription;
LogData += MyUpload.LogData;
ResponseStatusCode = MyUpload.ResponseStatusCode;
ResponseStatusDescription = MyUpload.ResponseStatusDescription;
ResponseHeaders = MyUpload.ResponseHeaders;
ResponseString = MyUpload.ResponseString;
if (RetBool == true)
{
break;
}
else
{
if (MyUpload.ResponseStatusCode == 503)
{
//A Service Unavailable response was returned. Wait and retry.
System.Threading.Thread.Sleep(1000 * i * i);
}
else if (MyUpload.ErrorNumber == 1003)
{
//Getting the response failed. This may be a network disconnection. Wait and retry.
System.Threading.Thread.Sleep(1000 * i * i);
}
else
{
//An error occured but retrying would not solve the problem.
break;
}
}
}
return RetBool;
}
private static int GetS3FileExists(String AWSAccessKeyId, String AWSSecretAccessKey, Boolean UseSSL, String RequestEndpoint, String BucketName, String UploadPrefix, int S3ErrorRetries)
{
Boolean RetBool;
int ErrorNumber = 0;
String ErrorDescription = "";
String LogData = "";
int ResponseStatusCode = 0;
String ResponseStatusDescription = "";
Dictionary<String, String> ResponseHeaders = new Dictionary<String, String>();
String ResponseString = "";
RetBool = MakeS3Request(AWSAccessKeyId, AWSSecretAccessKey, UseSSL, RequestEndpoint, BucketName, UploadPrefix, "", "HEAD", null, "", S3ErrorRetries, ref ErrorNumber, ref ErrorDescription, ref LogData, ref ResponseStatusCode, ref ResponseStatusDescription, ref ResponseHeaders, ref ResponseString);
if (RetBool == true)
{
return ResponseStatusCode;
}
else
{
WriteToLog("Error checking if the file exists on S3. UploadPrefix=" + UploadPrefix + " ResponseStatusCode=" + ResponseStatusCode + " ErrorDescription=" + ErrorDescription + " ResponseString=" + ResponseString);
return 0;
}
}
private static Dictionary<String, String> BuildDictionaryFromString(String InputString)
{
Dictionary<String, String> MyDictionary = new Dictionary<String, String>();
if (InputString.Contains(":") == true)
{
try
{
//Split multiple headers with the comma character.
String[] HeadersArray;
HeadersArray = InputString.Split(new Char[] { '|' });
String[] PairArray;
foreach (String ArrayValue in HeadersArray)
{
//Split the name and value with the colon character.
PairArray = ArrayValue.Split(new Char[] { ':' });
MyDictionary.Add(PairArray[0], PairArray[1]);
}
}
catch (Exception ex)
{
WriteToLog("Error parsing the UploadHeaders. ex.Message=" + ex.Message);
}
}
return MyDictionary;
}
private static String GetValueFromConfigOrArgument(string[] args, String ArgumentName)
{
String CurrentValue;
CurrentValue = System.Configuration.ConfigurationManager.AppSettings[ArgumentName];
ArgumentName = ArgumentName.ToLower();
String CurrentArg;
for (int i = 0; i < args.Length; i++)
{
CurrentArg = args[i].ToLower();
if (CurrentArg.StartsWith("-" + ArgumentName) | CurrentArg.StartsWith("/" + ArgumentName))
{
try
{
CurrentValue = args[i + 1];
}
catch (Exception ex)
{
WriteToLog("Error reading command line value. ArgumentName=" + ArgumentName + " ex.Message=" + ex.Message);
}
break;
}
}
return CurrentValue;
}
private static void MyUpload_ProgressChangedEvent()
{
if (ShowUploadProgress == true)
{
if (MyUpload.BytesTotal > 0)
{
SprightlySoftAWS.S3.Helper MyHelper = new SprightlySoftAWS.S3.Helper();
Double MyDouble = (Convert.ToDouble(MyUpload.BytesTransfered) / Convert.ToDouble(MyUpload.BytesTotal)) * 100;
Console.Write(MyDouble.ToString("0.##") + "% complete. " + MyUpload.CurrentKBpsSpeed.ToString("0.##") + " KB/s " + MyHelper.FormatByteSize(MyUpload.BytesTransfered) + " / " + MyHelper.FormatByteSize(MyUpload.BytesTotal) + " \r");
}
}
}
private static void SendEmailLog()
{
//Send the log information in an email.
if ((EmailLogCondition == 1 & ExitCode != 0) | EmailLogCondition == 2)
{
String EmailSubject;
if (ExitCode == 0)
{
EmailSubject = "SyncFromLocalToS3 Success, " + SoureFolder;
}
else
{
EmailSubject = "SyncFromLocalToS3 Failed, " + SoureFolder;
}
WriteToLog("");
WriteToLog("Sending the log email.");
SendEmail(EmailSmtpHost, EmailSmtpPort, EmailSmtpEnableSsl, EmailSmtpUserName, EmailSmtpPassword, EmailFromAddress, EmailRecipients, EmailSubject, EmailLogData);
}
}
private static void SendEmail(String EmailSmtpHost, int EmailSmtpPort, Boolean EmailSmtpEnableSsl, String EmailSmtpUserName, String EmailSmtpPassword, String EmailFromAddress, String EmailRecipients, String EmailSubject, String EmailBody)
{
try
{
System.Net.Mail.SmtpClient MySmtpClient = new System.Net.Mail.SmtpClient();
MySmtpClient.Host = EmailSmtpHost;
MySmtpClient.Port = EmailSmtpPort;
MySmtpClient.EnableSsl = EmailSmtpEnableSsl;
if (EmailSmtpUserName != "")
{
System.Net.NetworkCredential MyNetworkCredential = new System.Net.NetworkCredential();
MyNetworkCredential.UserName = EmailSmtpUserName;
MyNetworkCredential.Password = EmailSmtpPassword;
MySmtpClient.Credentials = MyNetworkCredential;
}
System.Net.Mail.MailAddress FromMailAddress = new System.Net.Mail.MailAddress(EmailFromAddress);
System.Net.Mail.MailAddress ToMailAddress = new System.Net.Mail.MailAddress(EmailRecipients);
System.Net.Mail.MailMessage MyMailMessage = new System.Net.Mail.MailMessage(FromMailAddress, ToMailAddress);
MyMailMessage.Subject = EmailSubject;
MyMailMessage.Body = EmailBody;
MySmtpClient.Send(MyMailMessage);
}
catch (Exception ex)
{
WriteToLog("Sending the log email failed. ex.Message=" + ex.Message);
ExitCode = 1;
}
}
private static void InitializeLogFile(Boolean OverwriteExistingLogFile)
{
if (LogFilePath != "")
{
String LogLine = "";
if (OverwriteExistingLogFile == true)
{
try
{
if (System.IO.File.Exists(LogFilePath) == true)
{
LogLine = "Deleting the existing log file. LogFilePath=" + LogFilePath;
System.IO.File.Delete(LogFilePath);
}
}
catch (Exception ex)
{
}
}
try
{
System.IO.StreamWriter MyStreamWriter;
MyStreamWriter = new System.IO.StreamWriter(LogFilePath, true, System.Text.Encoding.Unicode);
MyStreamWriter.Close();
if (LogLine != "")
{
WriteToLog(LogLine);
}
}
catch (Exception ex)
{
LogLine = "Error opening log file. No data will be written to the log file. LogFilePath=" + LogFilePath + " ex.Message=" + ex.Message;
Console.WriteLine(LogLine);
EmailLogData += LogLine + Environment.NewLine;
LogFilePath = "";
}
}
}
private static void WriteToLog(String LogLine, int LogLineLevel = 1)
{
if (LogLineLevel <= LogLevel)
{
Console.WriteLine(LogLine);
if (LogLevel > 1)
{
LogLine = DateTime.Now + " " + LogLine;
}
if (LogFilePath != "")
{
System.IO.StreamWriter MyStreamWriter;
MyStreamWriter = new System.IO.StreamWriter(LogFilePath, true, System.Text.Encoding.Unicode);
MyStreamWriter.WriteLine(LogLine);
MyStreamWriter.Close();
}
EmailLogData += LogLine + Environment.NewLine;
}
}
static void MyHandler(object sender, UnhandledExceptionEventArgs args)
{
Exception e = (Exception)args.ExceptionObject;
WriteToLog("");
WriteToLog("Error in application.");
WriteToLog("Message=" + e.Message);
WriteToLog("StackTrace=" + e.StackTrace);
ExitCode = 1;
//Send the log email.
SendEmailLog();
}
}
}
Submit an article or tip