Merge pull request #3 from henrikwidlund/test/github_pages

Restructure of actions and hosting

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @henrikwidlund

.github/workflows/ci.yml

@@ -1,54 +1,32 @@
-name: Publish
+name: Build and Run
 
 on:
- release:
- types: [published]
+ push:
+ branches:
+ - main
+ pull_request:
+ branches:
+ - main
 
 jobs:
- release:
- name: Release
- strategy:
- matrix:
- kind: ['linux', 'windows']
- include:
- - kind: linux
- os: ubuntu-latest
- target: linux-x64
- - kind: windows
- os: windows-latest
- target: win-x64
- runs-on: ${{ matrix.os }}
+ build:
+ runs-on: ubuntu-latest
+
  steps:
- - name: Checkout
- uses: actions/checkout@v1
+ - uses: actions/checkout@v2
 
- - name: Setup dotnet
+ - name: Setup .NET
  uses: actions/setup-dotnet@v1
  with:
  dotnet-version: 5.0.x
 
+ - name: Restore dependencies
+ run: dotnet restore
+
  - name: Build
- shell: bash
+ run: dotnet build -c Release --no-restore
+
+ - name: Run HostsParser
  run: |
- tag=$(git describe --tags --abbrev=0)
- release_name="HostsParser-$tag-${{ matrix.target }}"
- # Build everything
- dotnet publish --runtime "${{ matrix.target }}" -c Release -o "$release_name"
- # Pack files
- if [ "${{ matrix.target }}" == "win-x64" ]; then
- # Pack to zip for Windows
- 7z a -tzip "${release_name}.zip" "./${release_name}/HostsParser*" "./${release_name}/appsettings.json"
- else
- (cd "$release_name" && tar czvf "${release_name}.tar.gz" HostsParser*.dll HostsParser*.pdb HostsParser*.json appsettings.json)
- cp "${release_name}/${release_name}.tar.gz" .
- fi
- # Delete output directory
- rm -r "$release_name"
- - name: Publish
- uses: softprops/action-gh-release@v1
- with:
- files: |
- *.zip
- *.tar.gz
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ cd bin/Release/net5.0
+ dotnet HostsParser.dll

.github/workflows/publish-hosts.yml

@@ -1,9 +1,11 @@
-name: Build and publish hosts file
+name: Build and Publish
 
 on:
  push:
+ branches:
+ - main
  schedule:
- - cron: '0 */6 * * *' # Every hour
+ - cron: '0 */6 * * *' # Every 6 hours
 
 jobs:
  build:
@@ -24,23 +26,29 @@ jobs:
  run: dotnet build -c Release --no-restore
 
  - name: Run HostsParser
+ env:
+ USE_EXTERNAL_LAST_RUN: ${{ secrets.USE_EXTERNAL_LAST_RUN }}
  run: |
  cd bin/Release/net5.0
  dotnet HostsParser.dll
 
  - name: Copy hosts to root of branch
  run: |
+ mkdir ./public
  HOSTS=bin/Release/net5.0/hosts
  if test -f "$HOSTS"; then
- cp $HOSTS .
+ cp $HOSTS ./public
  fi
 
- APPSETTINGS=bin/Release/net5.0/appsettings.json
- if test -f "$APPSETTINGS"; then
- cp $APPSETTINGS .
+ MODIFIED=bin/Release/net5.0/MODIFIED.txt
+ if test -f "$MODIFIED"; then
+ cp $MODIFIED ./public
  fi
 
- - uses: EndBug/add-and-commit@v7
+ - name: Deploy
+ uses: peaceiris/actions-gh-pages@v3
+ if: github.ref == 'refs/heads/main'
  with:
- message: 'Auto update hosts'
- add: 'hosts appsettings.json --force'
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ publish_dir: ./public
+ keep_files: true

.github/workflows/release.yml

@@ -0,0 +1,54 @@
+name: Publish
+
+on:
+ release:
+ types: [published]
+
+jobs:
+ release:
+ name: Release
+ strategy:
+ matrix:
+ kind: ['linux', 'windows']
+ include:
+ - kind: linux
+ os: ubuntu-latest
+ target: linux-x64
+ - kind: windows
+ os: windows-latest
+ target: win-x64
+ runs-on: ${{ matrix.os }}
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v1
+
+ - name: Setup dotnet
+ uses: actions/setup-dotnet@v1
+ with:
+ dotnet-version: 5.0.x
+
+ - name: Build
+ shell: bash
+ run: |
+ tag=$(git describe --tags --abbrev=0)
+ release_name="HostsParser-$tag-${{ matrix.target }}"
+ # Build everything
+ dotnet publish --runtime "${{ matrix.target }}" -c Release -o "$release_name"
+ # Pack files
+ if [ "${{ matrix.target }}" == "win-x64" ]; then
+ # Pack to zip for Windows
+ 7z a -tzip "${release_name}.zip" "./${release_name}/HostsParser*" "./${release_name}/appsettings.json"
+ else
+ (cd "$release_name" && tar czvf "${release_name}.tar.gz" HostsParser*.dll HostsParser*.pdb HostsParser*.json appsettings.json)
+ cp "${release_name}/${release_name}.tar.gz" .
+ fi
+ # Delete output directory
+ rm -r "$release_name"
+ - name: Publish
+ uses: softprops/action-gh-release@v1
+ with:
+ files: |
+ *.zip
+ *.tar.gz
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Constants.cs

@@ -1,6 +1,6 @@
 namespace HostsParser
 {
- internal static class Constants
+ internal readonly struct Constants
  {
  internal const string PipeSign = "|";
  internal const string HatSign = "^";
@@ -13,5 +13,7 @@ internal static class Constants
  internal static readonly int IpFilterLength = IpFilter.Length;
  internal const string LoopbackEntry = "0.0.0.0 0.0.0.0";
  internal const string WwwPrefix = "www.";
+ internal const string UseExternalLastRun = "USE_EXTERNAL_LAST_RUN";
+ internal const string ModifiedFile = "MODIFIED.txt";
  }
 }

HostsParser.csproj

@@ -5,6 +5,7 @@
  <TargetFramework>net5.0</TargetFramework>
  <Copyright>Henrik Widlund</Copyright>
  <License>GPL-3.0 License</License>
+ <Nullable>enable</Nullable>
  </PropertyGroup>
 
  <ItemGroup>

Program.cs

@@ -28,11 +28,34 @@
 stopWatch.Start();
 
 var settings = JsonSerializer.Deserialize<Settings>(File.ReadAllBytes("appsettings.json"));
+if (settings == null)
+{
+ logger.LogError("Couldn't load settings. Terminating...");
+ return;
+}
 
 using var httpClient = new HttpClient();
 
+logger.LogInformation(WithTimeStamp("Start checking if external last run should be used"));
+if (settings.LastRunExternalUri != null
+ && bool.TryParse(Environment.GetEnvironmentVariable(Constants.UseExternalLastRun), out var useExternalLastRun)
+ && useExternalLastRun)
+{
+ var httpResponseMessage = await httpClient.GetAsync(settings.LastRunExternalUri);
+ if (httpResponseMessage.IsSuccessStatusCode)
+ {
+ var lastRunString = await httpResponseMessage.Content.ReadAsStringAsync();
+ if (long.TryParse(lastRunString, NumberStyles.Integer, NumberFormatInfo.InvariantInfo, out var lastRun))
+ {
+ settings = settings with { SourcePreviousUpdateEpoch = lastRun };
+ logger.LogInformation(WithTimeStamp($"Using external last run: {lastRunString}"));
+ }
+ }
+}
+logger.LogInformation(WithTimeStamp("Done checking if external last run should be used"));
+
 logger.LogInformation(WithTimeStamp("Start get source hosts"));
-var sourceUris = (await httpClient.GetStringAsync(settings!.SourceUri))
+var sourceUris = (await httpClient.GetStringAsync(settings.SourceUri))
  .Split(Constants.NewLine);
 logger.LogInformation(WithTimeStamp("Done get source hosts"));
 
@@ -42,21 +65,34 @@
  .Replace(" (UTC)", null);
 
 var modifiedDate = DateTime.Parse(modifiedDateString, DateTimeFormatInfo.InvariantInfo, DateTimeStyles.AssumeUniversal);
-var epoch = new DateTimeOffset(modifiedDate).ToUnixTimeSeconds();
+
+sourceUris = sourceUris
+ .Where(l => !l.StartsWith(Constants.HashSign))
+ .ToArray();
+
+logger.LogInformation(WithTimeStamp("Start get AdGuard hosts"));
+var adGuardLines = (await httpClient.GetStringAsync(settings.AdGuardUri))
+ .Split(Constants.NewLine);
+logger.LogInformation(WithTimeStamp("Done get AdGuard hosts"));
+
+modifiedDateString = adGuardLines[..6]
+ .Single(l => l.StartsWith("! Last modified: "))
+ .Replace("! Last modified: ", null);
+var adGuardModified = DateTime.Parse(modifiedDateString, DateTimeFormatInfo.InvariantInfo);
+
+var epoch = new DateTimeOffset(adGuardModified > modifiedDate ? adGuardModified : modifiedDate).ToUnixTimeSeconds();
+await File.WriteAllTextAsync(Constants.ModifiedFile, epoch.ToString(NumberFormatInfo.InvariantInfo));
 if (epoch <= settings.SourcePreviousUpdateEpoch)
 {
  logger.LogInformation(WithTimeStamp("Source not modified since previous run. Terminating..."));
  return;
 }
 
-logger.LogInformation(WithTimeStamp("Start get AdGuard hosts"));
-var adGuardLines = (await httpClient.GetStringAsync(settings.AdGuardUri))
- .Split(Constants.NewLine)
+adGuardLines = adGuardLines
  .Where(l => !string.IsNullOrWhiteSpace(l) && !l.StartsWith(Constants.ExclamationMark) && !l.StartsWith(Constants.AtSign))
  .Select(l => DnsUtilities.ReplaceAdGuard(l))
  .Where(l => !string.IsNullOrWhiteSpace(l))
  .ToArray();
-logger.LogInformation(WithTimeStamp("Done get AdGuard hosts"));
 
 logger.LogInformation(WithTimeStamp("Start combining host sources"));
 var combined = sourceUris
@@ -132,7 +168,7 @@
 
 logger.LogInformation(WithTimeStamp("Start building hosts results"));
 
-var newLines = new HashSet<string>(settings.HeaderLines);
+var newLines = new HashSet<string>(settings.HeaderLines) { $"! Lst Modified: {DateTime.UtcNow:u}" };
 foreach (var item in newLinesList)
  newLines.Add(item);
 

Settings.cs

@@ -8,5 +8,6 @@ internal record Settings(
  Uri AdGuardUri,
  string[] SkipLines,
  string[] HeaderLines,
- string[] KnownBadHosts);
+ string[] KnownBadHosts,
+ Uri? LastRunExternalUri);
 }

appsettings.json

@@ -1,6 +1,6 @@
 {
  "SourceUri": "https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-gambling-porn/hosts",
- "SourcePreviousUpdateEpoch": 1625160956,
+ "SourcePreviousUpdateEpoch": 0,
  "AdGuardUri": "https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt",
  "SkipLines": [
  "127.0.0.1 localhost",
@@ -47,5 +47,6 @@
  "adtech.fr",
  "adtech.us",
  "angelcities.com"
- ]
+ ],
+ "LastRunExternalUri": "https://henrikwidlund.github.io/HostsParser/MODIFIED.txt"
 }