珂珂的个人博客 - 一个程序猿的个人网站

阿里云同步工具

   

   为什么开发这个工具呢,因为自己网站的图片是存储在阿里云上的,但是后台管理确实直接传到本地的,不直接传到阿里云,因为网络不好,上传会很费时。

   阿里云有提供官方的sdk工具,不过没有达到我想要的效果,我就是简单的本地文件是什么结构阿里云上就是怎么样的。之前是按数据库来查询的,因为我上传文件数据库会有记录,还有个字段标志是否上传到阿里云,不过后来改善后不需要数据库,直接做成了文件系统同步。

   原理很简单,就是递归本地目录,记录文件的路径和MD5值,再去阿里云查询如果不存在则上传,如果MD5不一样也上传,如果阿里云有但是本地没有则可以删除阿里云的文件(可选)。

    代码还是有点多,我用WPF做的,需要特别注意多线程问题    

    xaml文件代码

<Window x:Class="Blogs.Tools.FileManager.AliyunSyncWindow"
        xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
        xmlns:con="clr-namespace:FYJ.Winui.Controls;assembly=FYJ.Winui"
        Title="阿里云同步工具" Height="400" Width="640">
    <Grid>
        <Grid.RowDefinitions>
            <RowDefinition Height="120"></RowDefinition>
            <RowDefinition Height="40"></RowDefinition>
            <RowDefinition Height="*"></RowDefinition>
            <RowDefinition Height="50"></RowDefinition>
        </Grid.RowDefinitions>
        <StackPanel Grid.Row="0" Margin="0,10,0,0">
            <WrapPanel>
                <TextBlock Text="本地目录:"></TextBlock>
                <TextBox Width="200" Margin="10,0,0,0" x:Name="txtLocalPath" Text="D:\Git\Blogs\Blogs.UI.Static\images-drive"></TextBox>
                <Button x:Name="btnBrower" Content="..." Click="btnBrower_Click" Margin="10,0,0,0"></Button>
            </WrapPanel>
            <WrapPanel Margin="0,10,0,0">
                <TextBlock Text="Bucket:"></TextBlock>
                <con:ComboBoxEx Width="200" Margin="10,0,0,0" x:Name="txtBucket"></con:ComboBoxEx>
            </WrapPanel>
            <WrapPanel Margin="0,10,0,0">
                <CheckBox Content="是否删除阿里云多余文件" Margin="10,0,0,0" x:Name="chkIsDelete"></CheckBox>
                <Button Content="同步文件到阿里云存储" Margin="10,0,0,0" x:Name="btnSyncAliyun" Click="btnSyncAliyun_Click"></Button>
                <Button Content="删除阿里云空目录" Margin="10,0,0,0"  x:Name="btnClearNullFolder" Click="btnClearNullFolder_Click"></Button>
                <TextBlock Text="错误数:" Margin="30,0,0,0"></TextBlock>
                <TextBlock  Text="0" Foreground="#FFF30707" Margin="10,0,0,0" x:Name="txtError"/>
            </WrapPanel>
        </StackPanel>
        <StackPanel Grid.Row="1">
            <ProgressBar  Height="40" x:Name="progressBar1"></ProgressBar>
        </StackPanel>
        <con:LogListBox Grid.Row="2" x:Name="logList1"></con:LogListBox>
        <TextBlock Grid.Row="3" Text="就绪" x:Name="txtStatus"></TextBlock>
    </Grid>
</Window>


后台代码

using Blogs.Tools.Util;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Data;
using System.Windows.Documents;
using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Shapes;

namespace Blogs.Tools.FileManager
{
    /// <summary>
    /// AliyunSyncWindow.xaml 的交互逻辑
    /// </summary>
    public partial class AliyunSyncWindow : Window
    {
        public AliyunSyncWindow()
        {
            InitializeComponent();
        }

        protected override void OnInitialized(EventArgs e)
        {
            base.OnInitialized(e);

            List<string> list = new List<string> { "images-drive", "album-drive", "files-drive" };
            this.txtBucket.ItemsSource = list;
            this.txtBucket.SelectedIndex = 0;
        }

        private void btnBrower_Click(object sender, RoutedEventArgs e)
        {
            System.Windows.Forms.FolderBrowserDialog diag = new System.Windows.Forms.FolderBrowserDialog();
            if (diag.ShowDialog() == System.Windows.Forms.DialogResult.OK)
            {
                txtLocalPath.Text = diag.SelectedPath;
            }
        }

        private async void btnSyncAliyun_Click(object sender, RoutedEventArgs e)
        {
            if (txtLocalPath.Text.Trim() == "")
            {
                MessageBox.Show("请选择本地路径");
                return;
            }

            if (txtBucket.Text.Trim() == "")
            {
                MessageBox.Show("请输入Bulcket");
                return;
            }

            if (System.IO.Path.GetFileName(txtLocalPath.Text) != txtBucket.Text)
            {
                MessageBox.Show("选择的Bulcket与路径不一致");
                return;
            }

            string localPatch = txtLocalPath.Text.Trim();
            string bucket = txtBucket.Text.Trim();
            bool isDelete = chkIsDelete.IsChecked.Value;
            await Task.Factory.StartNew(() =>
            {
                #region 处理同步
                //需要同步的文件列表
                this.Dispatcher.Invoke(() =>
                {
                    txtStatus.Text = "正在同步获取需要同步的信息...";
                });
                List<UploadMessage> syncList = new List<UploadMessage>();
                GetSyncList(bucket, "", syncList, localPatch);

                this.Dispatcher.Invoke(() => { this.txtError.Text = "0"; });

                this.Dispatcher.Invoke(() => { this.progressBar1.Maximum = syncList.Count; });

                for (int i = 0; i < syncList.Count; i++)
                {
                    UploadMessage up = syncList[i];

                    this.Dispatcher.Invoke(() =>
                    {
                        txtStatus.Text = "正在同步" + up.Key + "...";
                    });

                    if (up.IsDelete)
                    {
                        if (isDelete)
                        {
                            try
                            {
                                AliyunHelper.DeleteObject(bucket, up.Key);
                                this.logList1.AddItem("成功删除" + bucket + "/" + up.Key);
                            }
                            catch (Exception ex)
                            {
                                this.Dispatcher.Invoke(() =>
                                {
                                    this.logList1.AddItem("删除失败" + ex.Message);
                                    this.txtError.Text = (Convert.ToInt32(this.txtError.Text) + 1).ToString();
                                });

                            }
                        }
                    }
                    else
                    {
                        //上传文件到阿里云
                        FileStream stream = null;
                        try
                        {
                            stream = new FileStream(System.IO.Path.Combine(localPatch, up.Key), FileMode.Open);
                            AliyunHelper.PutObject(stream, bucket, up.Key);
                            this.Dispatcher.Invoke(() =>
                            {
                                this.logList1.AddItem("成功上传" + bucket + "/" + up.Key);
                            });

                        }
                        catch (Exception ex)
                        {
                            this.Dispatcher.Invoke(() =>
                            {
                                this.logList1.AddItem("上传失败" + ex.Message);
                                this.txtError.Text = (Convert.ToInt32(this.txtError.Text) + 1).ToString();
                            });

                        }
                        finally
                        {
                            if (stream != null)
                            {
                                stream.Close();
                            }
                        }
                    }
                    this.Dispatcher.Invoke(() =>
                    {
                        this.progressBar1.Value = i + 1;
                    });

                }
                #endregion
            });

            txtStatus.Text = "完成";
        }

        /// <summary>
        /// 获取需要同步的信息
        /// </summary>
        /// <param name="bucket"></param>
        /// <param name="key"></param>
        /// <param name="syncList"></param>
        /// <param name="rootLocalPath"></param>
        private void GetSyncList(string bucket, string key, List<UploadMessage> syncList, string rootLocalPath)
        {
            string aliyunEndpoint = System.Configuration.ConfigurationManager.AppSettings["aliyunEndpoint"];
            string aliyunKey = System.Configuration.ConfigurationManager.AppSettings["aliyunKey"];
            string aliyunSecret = System.Configuration.ConfigurationManager.AppSettings["aliyunSecret"];
            Aliyun.OpenServices.OpenStorageService.OssClient ossClient = new Aliyun.OpenServices.OpenStorageService.OssClient(aliyunEndpoint, aliyunKey, aliyunSecret);

            Aliyun.OpenServices.OpenStorageService.ListObjectsRequest request = new Aliyun.OpenServices.OpenStorageService.ListObjectsRequest(bucket);
            request.MaxKeys = 1000;
            request.Prefix = key;

            if (key != "")
            {
                request.Prefix = key.TrimEnd(new char[] { '\\', '/' }) + "/";
            }

            request.Delimiter = "/";
            var o = ossClient.ListObjects(request);
            var list = o.ObjectSummaries;

            List<string> localFiles = new List<string>();
            foreach (string file in Directory.GetFiles(System.IO.Path.Combine(rootLocalPath, key)))
            {
                FileMessage local = GetFileInfo(file, rootLocalPath);
                localFiles.Add(local.Key);
                //如果服务器上存在该文件并且MD5值相等
                if (list.Where(c => c.Key == local.Key && c.ETag == local.MD5).Count() == 1)
                {
                    continue;
                }
                else
                {
                    syncList.Add(new UploadMessage
                    {
                        Key = local.Key,
                        Bucket = bucket,
                        IsDelete = false,
                        MD5 = local.MD5
                    });
                }
            }

            foreach (var item in list)
            {
                //如果本地不包含服务器上的文件  则删除服务器上的文件
                if (!localFiles.Contains(item.Key))
                {
                    syncList.Add(new UploadMessage
                    {
                        Key = item.Key,
                        Bucket = bucket,
                        IsDelete = true,
                        MD5 = item.ETag
                    });
                }
            }

            //递归
            foreach (string folder in Directory.GetDirectories(System.IO.Path.Combine(rootLocalPath, key)))
            {
                GetSyncList(bucket, folder.Replace(rootLocalPath, "").TrimStart(new char[] { '/', '\\' }).Replace("\\", "/"), syncList, rootLocalPath);
            }
        }

        private FileMessage GetFileMessage(List<FileMessage> list, string key)
        {
            foreach (FileMessage fm in list)
            {
                if (fm.Key.Equals(key, StringComparison.CurrentCultureIgnoreCase))
                {
                    return fm;
                }
            }

            return null;
        }

        /// <summary>
        /// 获取文件信息
        /// </summary>
        private FileMessage GetFileInfo(String filePath, string baseDir)
        {
            FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
            long lenth = fs.Length;
            System.Security.Cryptography.HashAlgorithm algorithm = System.Security.Cryptography.MD5.Create();
            String result = BitConverter.ToString(algorithm.ComputeHash(fs)).Replace("-", "");
            fs.Close();
            return new FileMessage { Size = lenth, MD5 = result, Key = filePath.Replace(baseDir, "") };
        }

        private async void btnClearNullFolder_Click(object sender, RoutedEventArgs e)
        {
            if (txtBucket.Text.Trim() == "")
            {
                MessageBox.Show("请输入Bulcket");
                return;
            }

            string bucket = txtBucket.Text.Trim();
            //服务器文件信息
            List<FileMessage> folderList = new List<FileMessage>();
            List<FileMessage> fileList = new List<FileMessage>();
            txtStatus.Text = "正在处理...";
            await Task.Factory.StartNew(() =>
            {
                ClearNullFolder(bucket, "");
            });

            txtStatus.Text = "完成";
        }

        private void ClearNullFolder(string bucket, string key)
        {
            string aliyunEndpoint = System.Configuration.ConfigurationManager.AppSettings["aliyunEndpoint"];
            string aliyunKey = System.Configuration.ConfigurationManager.AppSettings["aliyunKey"];
            string aliyunSecret = System.Configuration.ConfigurationManager.AppSettings["aliyunSecret"];
            Aliyun.OpenServices.OpenStorageService.OssClient ossClient = new Aliyun.OpenServices.OpenStorageService.OssClient(aliyunEndpoint, aliyunKey, aliyunSecret);
            Aliyun.OpenServices.OpenStorageService.ListObjectsRequest request = new Aliyun.OpenServices.OpenStorageService.ListObjectsRequest(bucket);
            request.MaxKeys = 1000;
            request.Prefix = key;
            request.Delimiter = "/";
            var list = ossClient.ListObjects(request);

            if (list.CommonPrefixes.Count() == 0)
            {
                if (list.ObjectSummaries.Count() == 0
                    || (list.ObjectSummaries.Count() == 1 && list.ObjectSummaries.First().Key == key)
                    )
                {
                    try
                    {
                        AliyunHelper.DeleteObject(bucket, key);
                        this.logList1.AddItem("成功删除" + bucket + "/" + key);
                    }
                    catch (Exception ex)
                    {
                        this.logList1.AddItem("删除失败" + ex.Message);
                    }
                }
            }

            foreach (var v in list.CommonPrefixes)
            {
                ClearNullFolder(bucket, v);
            }
        }
    }

    public class FileMessage
    {
        public long Size { get; set; }

        public string MD5 { get; set; }

        private string _key;
        public string Key
        {
            get
            {
                return (_key == null ? null : _key.TrimStart('\\').TrimStart('/').Replace("\\", "/"));
            }
            set
            {
                _key = value;
            }
        }
    }

    public class UploadMessage
    {
        public long Size { get; set; }

        public string MD5 { get; set; }

        public string LocalPath { get; set; }

        public string Key { get; set; }

        public string Bucket { get; set; }

        public bool IsDelete { get; set; }
    }
}



上一篇:多线程域名查询工具

下一篇:个人代码全部开源


0 评论

查看所有评论

给个评论吧