0. 目录
- 参数介绍
- 断点续传和多线程下载的原理
- 源码
- 目录介绍
- 下载测试（多线程和断点续传）
5.1 下载
5.2 文件验证
1. 参数介绍
http请求和响应中，有几个header参数和请求/响应的大小有关：
- Range：这是一个RequestHeader，由客户端向服务器发起请求时携带，用来指定本次请求的文件范围（起始Byte位置和截止Byte位置），格式是 bytes=xx-yy，携带了这个header后，服务器将返回指定片段的文件内容，而非整个文件。
- Content-Length：这个一个ResponseHeader，在http响应中携带，用来表明本次返回的响应长度；
- Content-Range：这是一个ResponseHeader，在http响应中携带，用来表明本次返回的文件片段起止位置，以及整个文件的大小，只有请求Header里携带了Range，并且服务器支持的情况下，才会如此响应。
2. 断点续传和多线程下载的原理
通过Range请求，可以直接跳过已下载的数据，从断开的位置恢复下载，这样就实现了断点续传功能；如果服务器支持针对文件片段进行请求，在获取到资源Content-Length之后，就可以使用多线程同时请求多个片段，实现分片下载。
3. 源码
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using System.Net;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace downloader
{
enum DownloadState
{
Init,
Started,
Finished
}
class Program
{
public const string PathBase = @"C:\Lab\download";
public const string PathCaches = PathBase + @"\Caches";
//下载的块大小：4MB
public const int BlockSize = 1 << 22;
//下载线程数，固定值
public const int TaskThreads = 8;
static void Main(string[] args)
{
if (!Directory.Exists(PathCaches)) Directory.CreateDirectory(PathCaches);
List<TaskDownload> tasks = LoadTasks();
if (tasks.Count() == 0)
{
Dictionary<string, string> urls = new Dictionary<string, string>{
{"微信", "https://dl.softmgr.qq.com/original/im/WeChatSetup_2.9.0.123.exe" },
{"迅雷", "https://mythk.net/soft/ThunderSpeed1.0.34.360.exe" },
{"百度网盘", "http://wppkg.baidupcs.com/issue/netdisk/yunguanjia/BaiduNetdisk_6.9.7.4.exe" }
};
foreach (var kv in urls) tasks.Add(new TaskDownload(kv.Key, kv.Value));
Console.Title = $"创建新任务：{urls.Count}";
}
else
{
tasks.ForEach(t => t.Run(t.Segments.Count > 0 ? TaskThreads : 1));
Console.Title = $"加载历史任务：{tasks.Count}";
}
while (true)
{
Console.Clear();
Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss>") +
"\r\n" +
string.Join(
"\r\n\r\n",
tasks.Select(t => $"{t.Name}\t{t.ID} ({t.Segments.Sum(s => s.Transed) >> 10} KB of {t.Length >> 10} KB)\r\n{t.Message}")) +
"\r\n");
if (tasks.Count(t => !t.Finished) == 0) break;
Thread.Sleep(2000);
}
Console.WriteLine("完成。");
Console.ReadLine();
}
/// <summary>
/// 检查临时目录，如果有未完成的任务，加载到任务列表
/// </summary>
/// <returns></returns>
static List<TaskDownload> LoadTasks()
{//TaskSaveObject
List<TaskDownload> tasks = new List<TaskDownload>();
//遍历Cache目录，读取每个任务的config文件，根据文件内容恢复下载任务
foreach (string folder in Directory.GetDirectories(PathCaches))
{
TaskSaveObject taskConfig =
JsonConvert.DeserializeObject<TaskSaveObject>(File.ReadAllText(folder + @"\config"));
TaskDownload task = new TaskDownload(taskConfig.Name, taskConfig.Url, false)
{
ID = taskConfig.ID,
FileName = taskConfig.FileName,
Finished = false,
Length = taskConfig.Length,
Segments = new List<TaskSegment>()
};
foreach (SegmentSaveObject obj in taskConfig.Segments)
{
TaskSegment seg =
new TaskSegment(task, obj.From, obj.To, obj.SegID)
{
State = DownloadState.Init,
Transed = 0,
Message = "++++++++++"
};
FileInfo fInfo = new FileInfo($@"{PathCaches}\{seg.Task.ID}\{seg.SegID}");
if (fInfo.Exists)
{
//如果片段存在，且等于块大小，说明本片段下载完成，打上Finished标记
//如果未完成，删除临时文件重新下载
if (fInfo.Length == BlockSize)
{
seg.State = DownloadState.Finished;
seg.Transed = BlockSize;
seg.Message = "##########";
}
else
{
fInfo.Delete();
}
}
task.Segments.Add(seg);
}
tasks.Add(task);
}
return tasks;
}
public static string MD5(string plain) =>
BitConverter.ToString(
((HashAlgorithm)CryptoConfig.CreateFromName("MD5")).
ComputeHash(Encoding.UTF8.GetBytes(plain))
).Replace("-", "").Substring(8, 16);
}
class TaskDownload
{
//任务ID，唯一标识
public string ID { get; set; }
//分片任务列表
public List<TaskSegment> Segments =
new List<TaskSegment>();
public string Url { get; set; }
//保存的文件名
public string FileName { get; set; }
//任务名，用于界面显示，无实际用途
public string Name { get; set; }
//是否完成
public bool Finished = false;
//文件总长度 Bytes
public long Length { get; set; }
//已下载长度 Bytes
public long Transed = 0;
//进度提示信息
public string Message =>
string.Join("", Segments.Select(s => s.Message));
public string DataCache => $@"{Program.PathBase}\Caches\{ID}\";
public TaskDownload(string name, string url, bool create = true)
{
Name = name;
Url = url;
if (create)
{
string range = GetRange();
ID = Program.MD5(url);
FileName = Regex.Match(url, "(?<=/).*?$", RegexOptions.RightToLeft).Value;
//创建目录
if (Directory.Exists(DataCache)) Directory.Delete(DataCache, true);
Directory.CreateDirectory(DataCache);
if (string.IsNullOrEmpty(range))
{
//如果对Range无响应，使用单线程下载
Split(1);
}
else
{
Length = Convert.ToInt64(
Regex.Match(range, @"\d+$", RegexOptions.RightToLeft).Value);
Split(Program.TaskThreads);
}
}
}
/// <summary>
/// 下载失败时，用单线程重新下载
/// </summary>
public void Restart()
{
_ = GetRange();
Split(1);
}
/// <summary>
/// 下载2B长度内容，获取Content-Length和ETag
/// </summary>
/// <returns></returns>
public string GetRange()
{
using WebClient client = new WebClient();
client.Headers["Range"] = "bytes=0-1";
client.DownloadData(Url);
string range = client.ResponseHeaders["Content-Range"];
//ETag = client.ResponseHeaders["ETag"];
//if (string.IsNullOrEmpty(ETag)) ETag = client.ResponseHeaders["Last-Modified"];
return range;
}
public void Abort()
{
Segments.ForEach(s => s.Abort());
if (Directory.Exists(DataCache)) Directory.Delete(DataCache, true);
}
/// <summary>
/// 判断分片是否全部完成，如果全部完成，合并临时文件并输出
/// </summary>
public void Save()
{
if (Segments.Count(s => s.State != DownloadState.Finished) == 0)
{
Finished = true;
using FileStream stream = new FileStream(Path.Combine(Program.PathBase, FileName), FileMode.Append);
for (int i = 0; i < Segments.Count; i++)
{
byte[] seg = File.ReadAllBytes(Path.Combine(DataCache, $"{i}"));
stream.Write(seg, 0, seg.Length);
}
Directory.Delete(DataCache, true);
}
}
/// <summary>
/// 按指定数量对下载任务进行切片
/// </summary>
/// <param name="threads"></param>
void Split(int threads)
{
TaskSaveObject saveObj = new TaskSaveObject()
{
FileName = FileName,
ID = ID,
Length = Length,
Name = Name,
Url = Url
};
int taskCount = (int)Math.Ceiling(Length * 1.0 / Program.BlockSize);
for (int i = 0; i < taskCount; i++)
{
TaskSegment segment = new TaskSegment(
this,
i * Program.BlockSize,
Math.Min(Length, Program.BlockSize * (i + 1) - 1),
i);
Segments.Add(segment);
saveObj.Segments.Add(new SegmentSaveObject()
{
From = segment.From,
To = segment.To,
SegID = segment.SegID
});
}
File.WriteAllText(Path.Combine(DataCache, "config"), JsonConvert.SerializeObject(saveObj));
Run(threads);
}
public void Run(int threads)
{
Task.Run(() =>
{
while (Segments.Count(s => s.State == DownloadState.Init) > 0 &&
Segments.Count(s => s.State == DownloadState.Started) < threads)
{
Segments.Where(s => s.State == DownloadState.Init).First().Run();
Thread.Sleep(1000);
}
});
}
}
class TaskSegment
{
public TaskDownload Task { get; set; }
//分片任务ID
public int SegID { get; set; }
//起始字节
public long From { get; set; }
//结束字节
public long To { get; set; }
//分片任务已下载长度Bytes
public long Transed { get; set; }
public string Message { get; set; }
public DownloadState State { get; set; }
WebClient client = new WebClient();
//为了方便解除事件，不使用匿名委托
DownloadProgressChangedEventHandler handlerDownloading;
AsyncCompletedEventHandler handlerDownloadCompleted;
public TaskSegment(TaskDownload task, long from, long to, int id)
{
Task = task;
From = from;
To = to;
SegID = id;
State = DownloadState.Init;
Message = To > 0 ? $"++++++++++" : "单线程：0KB";
}
public void Run()
{
State = DownloadState.Started;
client.Headers["Range"] = $"bytes={From}-" + (To == 0 ? "" : $"{To}");
client.DownloadFileAsync(new Uri(Task.Url), Path.Combine(Task.DataCache, $"{SegID}"));
handlerDownloading = (s, e) =>
{
Transed = e.BytesReceived;
int percentage = e.ProgressPercentage / 10;
Message = To > 0 ?
new string('#', percentage) + new string('+', 10 - percentage) :
$"单线程下载：{Transed >> 10}KB";
};
client.DownloadProgressChanged += handlerDownloading;
handlerDownloadCompleted = (s, e) =>
{
State = DownloadState.Finished;
client.DownloadProgressChanged -= handlerDownloading;
client.DownloadFileCompleted -= handlerDownloadCompleted;
client.Dispose();
client = null;
Task.Save();
Message = $"##########";
};
client.DownloadFileCompleted += handlerDownloadCompleted;
}
public void Abort()
{
client.DownloadProgressChanged -= handlerDownloading;
client.DownloadFileCompleted -= handlerDownloadCompleted;
client.CancelAsync();
client.Dispose();
client = null;
File.Delete(Path.Combine(Task.DataCache, $"{SegID}"));
}
}
/// <summary>
/// 摘录关键的任务信息和分片信息，用于保存下载任务的状态，供恢复时使用
/// </summary>
class TaskSaveObject
{
public string ID { get; set; }
public string Url { get; set; }
public string FileName { get; set; }
public string Name { get; set; }
public long Length { get; set; }
public List<SegmentSaveObject> Segments =
new List<SegmentSaveObject>();
}
/// <summary>
/// 摘录关键的任务信息和分片信息，用于保存下载任务的状态，供恢复时使用
/// </summary>
class SegmentSaveObject
{
public int SegID { get; set; }
//起始字节位置
public long From { get; set; }
//结束字节位置
public long To { get; set; }
}
}4. 目录介绍
5. 下载测试（多线程和断点续传）
5.1 下载
5.2 文件验证
