From 5a748da2806f24d67d3cc30e1852270bee4c94f4 Mon Sep 17 00:00:00 2001 From: the1812 Date: Thu, 1 Nov 2018 16:16:59 +0800 Subject: [PATCH] Open source code of project builder --- .gitignore | 2 +- builder/builder-config.json | 8 + builder/dotnet/BuildCache.cs | 91 +++++++++++ builder/dotnet/BuilderConfig.cs | 16 ++ builder/dotnet/DarkStyleBuilder.cs | 21 +++ builder/dotnet/Extensions.cs | 18 +++ builder/dotnet/MasterBuilder.cs | 32 ++++ builder/dotnet/OfflineBuilder.cs | 114 ++++++++++++++ builder/dotnet/PreviewBuilder.cs | 21 +++ builder/dotnet/Program.cs | 43 +++++ builder/dotnet/ProjectBuilder.cs | 47 ++++++ builder/dotnet/ResourceBuilder.cs | 241 +++++++++++++++++++++++++++++ builder/dotnet/dotnet.csproj | 18 +++ 13 files changed, 671 insertions(+), 1 deletion(-) create mode 100644 builder/builder-config.json create mode 100644 builder/dotnet/BuildCache.cs create mode 100644 builder/dotnet/BuilderConfig.cs create mode 100644 builder/dotnet/DarkStyleBuilder.cs create mode 100644 builder/dotnet/Extensions.cs create mode 100644 builder/dotnet/MasterBuilder.cs create mode 100644 builder/dotnet/OfflineBuilder.cs create mode 100644 builder/dotnet/PreviewBuilder.cs create mode 100644 builder/dotnet/Program.cs create mode 100644 builder/dotnet/ProjectBuilder.cs create mode 100644 builder/dotnet/ResourceBuilder.cs create mode 100644 builder/dotnet/dotnet.csproj diff --git a/.gitignore b/.gitignore index 3e8803705..4c38cb4b2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ .vscode/ -builder/ +builder/node/ build-scripts/ .node_modules/ package-lock.json diff --git a/builder/builder-config.json b/builder/builder-config.json new file mode 100644 index 000000000..8dbc176a0 --- /dev/null +++ b/builder/builder-config.json @@ -0,0 +1,8 @@ +{ + "preview": "bilibili-evolved.preview.user.js", + "offline": "bilibili-evolved.offline.user.js", + "previewOffline": "bilibili-evolved.preview-offline.user.js", + "master": "bilibili-evolved.user.js", + "logoPath": "images/logo.png", + "owner": "the1812" +} \ No newline at end of file diff --git a/builder/dotnet/BuildCache.cs b/builder/dotnet/BuildCache.cs new file mode 100644 index 000000000..35a1524fd --- /dev/null +++ b/builder/dotnet/BuildCache.cs @@ -0,0 +1,91 @@ +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Linq; + +namespace BilibiliEvolved.Build +{ + class BuildCache : IDisposable + { + private readonly string fileName; + private readonly ConcurrentDictionary sha1Cache = new ConcurrentDictionary(); + private readonly DirectoryInfo cacheDirectory; + private readonly SHA1Managed sha1; + + public BuildCache() : this(Environment.CurrentDirectory) {} + public BuildCache(string path) : this(new DirectoryInfo(path)) {} + public BuildCache(DirectoryInfo path) + { + cacheDirectory = path; + fileName = Path.Combine(cacheDirectory.FullName, "build.cache"); + sha1 = new SHA1Managed(); + LoadCache(); + } + + private string hashToString(byte[] hash) + { + return string.Join("", hash.Select(b => b.ToString("X2")).ToArray()); + } + private string getFileHashString(string path) + { + var bytes = File.ReadAllBytes(path); + return hashToString(sha1.ComputeHash(bytes)); + } + + public void LoadCache() + { + if (File.Exists(fileName)) + { + var lines = File.ReadAllText(fileName) + .Split(Environment.NewLine) + .Where(l => !string.IsNullOrWhiteSpace(l)); + lines.ForEach(line => + { + var data = line.Split("|"); + var fileName = data[0]; + var sha1Text = data[1]; + sha1Cache.TryAdd(fileName, sha1Text); + }); + } + } + public void SaveCache() + { + var builder = new StringBuilder(); + sha1Cache.ForEach(pair => + { + builder + .Append(pair.Key) + .Append("|") + .Append(pair.Value) + .Append(Environment.NewLine); + }); + File.WriteAllText(fileName, builder.ToString().Trim()); + } + + public void AddCache(string file) + { + var hashText = getFileHashString(file); + sha1Cache.AddOrUpdate(file, hashText, (k, v) => hashText); + } + public bool Contains(string file) + { + var hashText = getFileHashString(file); + if (sha1Cache.ContainsKey(file)) + { + return sha1Cache[file] == hashText; + } + else + { + return false; + } + } + + public void Dispose() + { + sha1?.Dispose(); + } + } +} diff --git a/builder/dotnet/BuilderConfig.cs b/builder/dotnet/BuilderConfig.cs new file mode 100644 index 000000000..adf2612ca --- /dev/null +++ b/builder/dotnet/BuilderConfig.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BilibiliEvolved.Build +{ + class BuilderConfig + { + public string Master { get; set; } + public string Preview { get; set; } + public string Offline { get; set; } + public string PreviewOffline { get; set; } + public string LogoPath { get; set; } + public string Owner { get; set; } + } +} diff --git a/builder/dotnet/DarkStyleBuilder.cs b/builder/dotnet/DarkStyleBuilder.cs new file mode 100644 index 000000000..b00d73354 --- /dev/null +++ b/builder/dotnet/DarkStyleBuilder.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Linq; + +namespace BilibiliEvolved.Build +{ + partial class ProjectBuilder + { + public ProjectBuilder BuildDarkStyles() + { + var files = new DirectoryInfo("min").EnumerateFiles().Where(f => f.FullName.Contains("dark-slice")); + var fullStyle = files.Select(f => File.ReadAllText(f.FullName)) + .Aggregate((acc, s) => acc + s); + File.WriteAllText("min/dark.min.scss", fullStyle); + WriteSuccess("Dark style build complete."); + return this; + } + } +} diff --git a/builder/dotnet/Extensions.cs b/builder/dotnet/Extensions.cs new file mode 100644 index 000000000..b623c7ebc --- /dev/null +++ b/builder/dotnet/Extensions.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace BilibiliEvolved.Build +{ + static class Extensions + { + public static void ForEach(this IEnumerable collection, Action action) + { + foreach (var item in collection) + { + action(item); + } + } + } +} diff --git a/builder/dotnet/MasterBuilder.cs b/builder/dotnet/MasterBuilder.cs new file mode 100644 index 000000000..c7b224d88 --- /dev/null +++ b/builder/dotnet/MasterBuilder.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; + +namespace BilibiliEvolved.Build +{ + partial class ProjectBuilder + { + private readonly Regex ownerRegex = new Regex(@"(github\.com/)(.+)(/Bilibili-Evolved)"); + public ProjectBuilder BuildMaster() + { + var master = Source; + master = ownerRegex.Replace(master, "${1}" + config.Owner + "${3}"); + var replaceMap = new Dictionary + { + { @"Bilibili-Evolved/preview", @"Bilibili-Evolved/master" }, + { @"Bilibili-Evolved/raw/preview", @"Bilibili-Evolved/raw/master" }, + { SourcePath, OutputPath }, + { @"// settings.guiSettings = true;", @"settings.guiSettings = true;" }, + { @"Bilibili Evolved (Preview)", @"Bilibili Evolved" }, + { @"增强哔哩哔哩Web端体验(预览版分支):", @"增强哔哩哔哩Web端体验:" }, + { @"settings.debug = true;", @"settings.debug = false;" }, + }; + replaceMap.ForEach(item => master = master.Replace(item.Key, item.Value)); + Output = master; + WriteSuccess("Master build complete."); + return this; + } + } +} diff --git a/builder/dotnet/OfflineBuilder.cs b/builder/dotnet/OfflineBuilder.cs new file mode 100644 index 000000000..cfd02e718 --- /dev/null +++ b/builder/dotnet/OfflineBuilder.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +namespace BilibiliEvolved.Build +{ + partial class ProjectBuilder + { + private string offlineText; + private string offlineVersion; + + private void replaceInfo(Dictionary map) + { + var logoBytes = File.ReadAllBytes(config.LogoPath); + var logoBase64 = $"data:image/png;base64,{Convert.ToBase64String(logoBytes)}"; + map.ForEach(item => offlineText = offlineText.Replace(item.Key, item.Value)); + offlineText = ownerRegex.Replace(offlineText, "${1}" + config.Owner + "${3}"); + offlineText = offlineText.Replace( + $"@icon https://raw.githubusercontent.com/{config.Owner}/Bilibili-Evolved/master/images/logo.png", + $"@icon {logoBase64}"); + } + private void generateVersion() + { + var startDate = new DateTime(2018, 7, 16, 15, 46, 53); + var versionRegex = new Regex(@"(@version[ ]*)[\d\.]*"); + var version = DateTime.Now.ToOADate() - startDate.ToOADate(); + offlineVersion = version.ToString("0.00"); + offlineText = versionRegex.Replace(offlineText, "${1}" + offlineVersion); + } + private void compileOfflineData() + { + var onlineRoot = new Regex(@"Resource.root = ""(.*)"";").Match(offlineText).Groups[1].Value; + var urlList = (from match in new Regex(@"path:\s*""(.*)""").Matches(offlineText) + as IEnumerable + select match.Groups[1].Value.Trim() + ).ToList(); + + var downloadCodeStart = @"// \+#Offline build placeholder"; + var downloadCodeEnd = @"// \-#Offline build placeholder"; + var downloadCodes = new Regex($"({downloadCodeStart}([^\0]*){downloadCodeEnd})").Match(offlineText).Groups[0].Value; + + var offlineData = "const offlineData = {};" + Environment.NewLine; + foreach (var url in urlList) + { + var text = File.ReadAllText(url); + if (url.EndsWith(".js")) + { + offlineData = offlineData + $"offlineData[\"{onlineRoot + url}\"] = {text}" + Environment.NewLine; + } + else + { + offlineData = offlineData + $"offlineData[\"{onlineRoot + url}\"] = `{text}`;" + Environment.NewLine; + } + } + offlineText = offlineText + .Replace(@"// [Offline build placeholder]", offlineData) + .Replace(downloadCodes, "this.text=this.type.preprocessor(offlineData[this.url]);resolve(this.text);"); + } + private void buildFile(string path) + { + if (File.Exists(path)) + { + var offlineFileText = File.ReadAllText(path); + + var noVersion = new Regex(@"// @version[ ]*(.*)" + Environment.NewLine); + var originalOffline = noVersion.Replace(offlineFileText, ""); + var currentOffline = noVersion.Replace(offlineText, ""); + + if (currentOffline == originalOffline) + { + offlineVersion = noVersion.Match(offlineFileText).Groups[1].Value.Trim(); + return; + } + } + + File.WriteAllText(path, offlineText); + } + private ProjectBuilder build(Dictionary replaceMap, string outputPath, string successMessage) + { + offlineText = Output; + replaceInfo(replaceMap); + generateVersion(); + compileOfflineData(); + buildFile(outputPath); + + WriteSuccess(successMessage); + return this; + } + + public ProjectBuilder BuildOffline() + { + var replaceMap = new Dictionary + { + { "Bilibili Evolved", "Bilibili Evolved (Offline)" }, + { "增强哔哩哔哩Web端体验:", "增强哔哩哔哩Web端体验(离线版):" }, + { $"master/{config.Master}", $"master/{config.Offline}" }, + }; + return build(replaceMap, config.Offline, "Offline build complete."); + } + public ProjectBuilder BuildPreviewOffline() + { + var replaceMap = new Dictionary + { + { "Bilibili Evolved", "Bilibili Evolved (Preview Offline)" }, + { "增强哔哩哔哩Web端体验:", "增强哔哩哔哩Web端体验(预览离线版):" }, + { $"master/{config.Master}", $"preview/{config.PreviewOffline}" }, + }; + return build(replaceMap, config.PreviewOffline, "Preview Offline build complete."); + } + } +} diff --git a/builder/dotnet/PreviewBuilder.cs b/builder/dotnet/PreviewBuilder.cs new file mode 100644 index 000000000..080443287 --- /dev/null +++ b/builder/dotnet/PreviewBuilder.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; + +namespace BilibiliEvolved.Build +{ + partial class ProjectBuilder + { + public ProjectBuilder BuildPreview() + { + var version = new Regex(@"//[ ]*@version[ ]*(.+)") + .Match(Source).Groups[1].Value.Trim(); + Source = ownerRegex.Replace(Source, "${1}" + config.Owner + "${3}"); + File.WriteAllText(SourcePath, Source); + File.WriteAllText("version.txt", version); + return this; + } + } +} diff --git a/builder/dotnet/Program.cs b/builder/dotnet/Program.cs new file mode 100644 index 000000000..efeced38b --- /dev/null +++ b/builder/dotnet/Program.cs @@ -0,0 +1,43 @@ +using System; +using System.Linq; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; +using System.Collections.Generic; +using System.Diagnostics; +using Microsoft.Extensions.Configuration; + +namespace BilibiliEvolved.Build +{ + class Program + { + public static void Main(string[] args) + { + try + { + var configFile = new ConfigurationBuilder() + .AddJsonFile( + Path.Combine(Environment.CurrentDirectory, "builder/builder-config.json"), + optional: false, + reloadOnChange: false) + .Build(); + var config = new BuilderConfig(); + configFile.Bind(config); + + var builder = new ProjectBuilder(config); + builder + .BuildPreview() + .BuildMaster() + .BuildResources() + .BuildPreviewOffline() + .BuildOffline() + .BuildFinalOutput(); + } + catch (Exception ex) + { + Console.ForegroundColor = ConsoleColor.Red; + Console.Error.WriteLine($"Unexcepted Error: {ex.Message}"); + } + } + } +} \ No newline at end of file diff --git a/builder/dotnet/ProjectBuilder.cs b/builder/dotnet/ProjectBuilder.cs new file mode 100644 index 000000000..1cc601b31 --- /dev/null +++ b/builder/dotnet/ProjectBuilder.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace BilibiliEvolved.Build +{ + partial class ProjectBuilder + { + public ProjectBuilder(BuilderConfig config) + { + this.config = config; + SourcePath = config.Preview; + Source = File.ReadAllText(SourcePath); + WriteInfo("[Bilibili Evolved] Project builder started."); + WriteInfo($"Working directory: {Environment.CurrentDirectory}"); + WriteInfo(); + } + private BuilderConfig config; + public double MinifiedResourceLength { get; set; } + public double OriginalResourceLength { get; set; } + public string Source { get; private set; } + public string Output { get; private set; } + public string SourcePath { get; private set; } + public string OutputPath { get; set; } = "bilibili-evolved.user.js"; + public void BuildFinalOutput() + { + var ratio = 100.0 * MinifiedResourceLength / OriginalResourceLength; + File.WriteAllText(OutputPath, Output); + WriteInfo(); + WriteHint($"External resource size -{(100.0 - ratio):0.##}%"); + WriteInfo("Build complete.", ConsoleColor.Green); + } + public void WriteInfo(string message = "", ConsoleColor color = ConsoleColor.Gray) + { + lock (this) + { + Console.ForegroundColor = color; + Console.WriteLine(message); + } + } + public void WriteSuccess(string message) => WriteInfo(message, ConsoleColor.Blue); + public void WriteError(string message) => WriteInfo(message, ConsoleColor.Red); + public void WriteHint(string message) => WriteInfo(message, ConsoleColor.DarkGray); + } +} diff --git a/builder/dotnet/ResourceBuilder.cs b/builder/dotnet/ResourceBuilder.cs new file mode 100644 index 000000000..af3ad20c2 --- /dev/null +++ b/builder/dotnet/ResourceBuilder.cs @@ -0,0 +1,241 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace BilibiliEvolved.Build +{ + partial class ProjectBuilder + { + public ProjectBuilder BuildResources() + { + new CssMinifier().Build(this); + new HtmlMinifier().Build(this); + new JavascriptMinifier().Build(this); + return this; + } + } + abstract class ResourceMinifier + { + public abstract Predicate FileFilter { get; } + public abstract string ResouceType { get; } + public abstract string Minify(string input); + protected string GetMinifiedFileName(string path) + { + var fileInfo = new FileInfo(path); + return "min/" + fileInfo.Name.Insert(fileInfo.Name.LastIndexOf("."), ".min"); + } + protected IEnumerable GetFiles(Predicate filter) + { + string getRelativePath(string fullPath) + { + var currentFolder = Environment.CurrentDirectory; + var path = fullPath.Replace(currentFolder, ""); + if (path.StartsWith(Path.DirectorySeparatorChar)) + { + return path.Remove(path.IndexOf(Path.DirectorySeparatorChar), 1); + } + else + { + return path; + } + } + IEnumerable getFiles(Predicate predicate, string path) + { + var list = new List(); + var currentDirectory = new DirectoryInfo(path); + list.AddRange(currentDirectory.EnumerateFiles() + .Where(file => predicate(file)) + .Select(file => getRelativePath(file.FullName))); + foreach (var subDir in currentDirectory.EnumerateDirectories()) + { + list.AddRange(getFiles(filter, subDir.FullName)); + } + return list; + } + var directory = Environment.CurrentDirectory; + if (!directory.EndsWith(Path.DirectorySeparatorChar)) + { + directory = directory + Path.DirectorySeparatorChar; + } + return getFiles(filter, directory); + } + public virtual ProjectBuilder Build(ProjectBuilder builder) + { + var files = GetFiles(file => + FileFilter(file) + && !file.FullName.Contains(@".vs\") + && !file.FullName.Contains(@".vscode\") + && !file.FullName.Contains(@"build-scripts\") + && !file.FullName.Contains(@"node_modules\") + ); + using (var cache = new BuildCache()) + { + var changedFiles = files.Where(file => !cache.Contains(file)); + Parallel.ForEach(changedFiles, path => + { + builder.WriteInfo($"{ResouceType} minify: {path}"); + + var text = File.ReadAllText(path); + var result = Minify(text); + + var outputPath = GetMinifiedFileName(path); + File.WriteAllText(outputPath, result); + cache.AddCache(path); + + builder.WriteHint($"\t=> {outputPath.PadRight(48)}{(100.0 * result.Length / text.Length):0.##}%"); + }); + cache.SaveCache(); + } + files.ForEach(file => + { + builder.OriginalResourceLength += new FileInfo(file).Length; + builder.MinifiedResourceLength += new FileInfo(GetMinifiedFileName(file)).Length; + }); + builder.WriteSuccess($"{ResouceType} minify complete."); + return builder; + } + } + sealed class CssMinifier : ResourceMinifier + { + public override Predicate FileFilter { get; } = file => + { + return !file.FullName.Contains(".min") + && !file.FullName.Contains("dark.scss") + && !file.FullName.Contains("dark-template") + && (file.Extension == ".css" || file.Extension == ".scss"); + }; + + public override string ResouceType { get; } = "CSS"; + + public override string Minify(string input) + { + var commentRegex = new Regex(@"/\*[^\0]*?\*/|^\s*//.*$", RegexOptions.Multiline); + input = commentRegex.Replace(input, ""); + + var selectorRegex = new Regex(@"[\s]*([^\0,}]+?)[\s]*({)|[\s]*([^\0}]+?)(,)" + Environment.NewLine); + input = selectorRegex.Replace(input, "$1$2$3$4"); + + var ruleRegex = new Regex(@"[\s]*([a-z\-]+:)[ ]*(.*?)[ ]*(!important)?;[\s]*"); + input = ruleRegex.Replace(input, "$1$2$3;"); + + return input + .Replace(Environment.NewLine, "") + .Replace("\n", "") + .Replace("\r", "") + .Replace(", ", ","); + } + + public override ProjectBuilder Build(ProjectBuilder builder) + { + base.Build(builder); + return builder.BuildDarkStyles(); + } + } + sealed class JavascriptMinifier : ResourceMinifier + { + private static readonly string UglifyEsArguments = @"-m"; + private static readonly string NodePath = "node"; + + private static string uglifyEsAbsolutePath = null; + private static string UglifyEsAbsolutePath + { + get + { + if (uglifyEsAbsolutePath is null) + { + uglifyEsAbsolutePath = getUglifyEsPath(); + } + return uglifyEsAbsolutePath; + } + set => uglifyEsAbsolutePath = value; + } + private static string getUglifyEsPath() + { + var uglifyEsPath = @"\uglify-es\bin\uglifyjs"; + var localPath = @"\node_modules" + uglifyEsPath; + var globalPath = Environment.GetEnvironmentVariable("AppData") + @"\npm\node_modules" + uglifyEsPath; + + if (File.Exists(localPath)) + { + return localPath; + } + else if (File.Exists(globalPath)) + { + return globalPath; + } + else + { + return ""; + } + } + + public JavascriptMinifier() + { + if (!File.Exists(UglifyEsAbsolutePath)) + { + throw new FileNotFoundException("Node.js module \"uglify-es\" not found."); + } + } + + public override Predicate FileFilter { get; } = file => + { + return !file.FullName.Contains(".min") + && !file.FullName.Contains(@"builder\") + && !file.FullName.Contains("bilibili-evolved.") + && file.Extension == ".js"; + }; + + public override string ResouceType { get; } = "JavaScript"; + + public override string Minify(string input) + { + var processInfo = new ProcessStartInfo + { + FileName = NodePath, + Arguments = $"{UglifyEsAbsolutePath} {UglifyEsArguments}", + UseShellExecute = false, + RedirectStandardInput = true, + RedirectStandardOutput = true, + }; + var process = Process.Start(processInfo); + using (var writer = new StreamWriter(process.StandardInput.BaseStream, Encoding.UTF8)) + { + writer.Write(input); + writer.Flush(); + writer.Close(); + using (var reader = new StreamReader(process.StandardOutput.BaseStream, Encoding.UTF8)) + { + return reader.ReadToEnd().Trim(); + } + } + } + } + sealed class HtmlMinifier : ResourceMinifier + { + public override Predicate FileFilter { get; } = file => + { + return !file.FullName.Contains(".min") && file.Extension == ".html"; + }; + + public override string ResouceType { get; } = "HTML"; + + public override string Minify(string input) + { + var commentRegex = new Regex(@"", RegexOptions.Multiline); + input = commentRegex.Replace(input, ""); + + var blankRegex = new Regex(@"(?<=>)[\s]*([^\s]*)[\s]*(?=<)|[\s]*(?=/>)", RegexOptions.Multiline); + input = blankRegex.Replace(input, "$1"); + + return input + .Replace(Environment.NewLine, "") + .Replace("\n", "") + .Replace("\r", ""); + } + } +} diff --git a/builder/dotnet/dotnet.csproj b/builder/dotnet/dotnet.csproj new file mode 100644 index 000000000..fed16a8c3 --- /dev/null +++ b/builder/dotnet/dotnet.csproj @@ -0,0 +1,18 @@ + + + + Exe + netcoreapp2.1 + Grant Howard + Bilibili-Evolved Project Builder + BilibiliEvolved.Build.Program + build + + + + + + + + +