build: first commit

.editorconfig

@@ -0,0 +1,19 @@
+# https://editorconfig.org
+
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+max_line_length = 80
+indent_brace_style = 1TBS
+spaces_around_operators = true
+quote_type = auto
+
+[package.json]
+indent_style = space
+indent_size = 2

.gitattributes

@@ -0,0 +1 @@
+* text=auto

.github/dependabot.yml

@@ -0,0 +1,11 @@
+version: 2
+updates:
+ - package-ecosystem: npm
+ directory: '/'
+ schedule:
+ interval: daily
+ - package-ecosystem: 'github-actions'
+ directory: '/'
+ schedule:
+ # Check for updates to GitHub Actions every weekday
+ interval: 'daily'

.github/workflows/cron.yml

@@ -0,0 +1,28 @@
+name: cron
+
+on:
+ schedule:
+ # Cron job every day at 12:00
+ # https://crontab.guru/#0_12_*_*_*
+ - cron: '0 0 * * *'
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ with:
+ token: ${{ secrets.GH_TOKEN }}
+ fetch-depth: 2
+ - name: Setup Node.js
+ uses: actions/setup-node@v3
+ with:
+ node-version: lts/*
+ - name: Install
+ run: npm install --no-package-lock
+ - name: Cron
+ run: |
+ git config --global user.email ${{ secrets.GIT_EMAIL }}
+ git config --global user.name ${{ secrets.GIT_USERNAME }}
+ npm run cronjob

.github/workflows/main.yml

@@ -0,0 +1,65 @@
+name: main
+
+on:
+ push:
+ branches:
+ - master
+
+jobs:
+ contributors:
+ if: "${{ github.event.head_commit.message != 'build: contributors' }}"
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+ token: ${{ secrets.GITHUB_TOKEN }}
+ - name: Setup Node.js
+ uses: actions/setup-node@v3
+ with:
+ node-version: lts/*
+ - name: Contributors
+ run: |
+ git config --global user.email ${{ secrets.GIT_EMAIL }}
+ git config --global user.name ${{ secrets.GIT_USERNAME }}
+ npm run contributors
+ - name: Push changes
+ run: |
+ git push origin ${{ github.head_ref }}
+
+ release:
+ if: |
+ !startsWith(github.event.head_commit.message, 'chore(release):') &&
+ !startsWith(github.event.head_commit.message, 'docs:') &&
+ !startsWith(github.event.head_commit.message, 'ci:')
+ needs: [contributors]
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
+ - name: Setup Node.js
+ uses: actions/setup-node@v3
+ with:
+ node-version: lts/*
+ - name: Install
+ run: npm install --no-package-lock
+ - name: Test
+ run: npm test
+ # - name: Report
+ # run: mkdir -p coverage && npx c8 report --reporter=text-lcov > coverage/lcov.info
+ # - name: Coverage
+ # uses: coverallsapp/github-action@main
+ # with:
+ # github-token: ${{ secrets.GITHUB_TOKEN }}
+ - name: Release
+ env:
+ CONVENTIONAL_GITHUB_RELEASER_TOKEN: ${{ secrets.GH_TOKEN }}
+ NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+ run: |
+ git config --global user.email ${{ secrets.GIT_EMAIL }}
+ git config --global user.name ${{ secrets.GIT_USERNAME }}
+ git pull origin master
+ npm run release

.github/workflows/pull_request.yml

@@ -0,0 +1,33 @@
+name: pull_request
+
+on:
+ push:
+ branches:
+ - master
+ pull_request:
+ branches:
+ - master
+
+jobs:
+ test:
+ if: github.ref != 'refs/heads/master'
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
+ - name: Setup Node.js
+ uses: actions/setup-node@v3
+ with:
+ node-version: lts/*
+ - name: Install
+ run: npm install --no-package-lock
+ - name: Test
+ run: npm test
+ # - name: Report
+ # run: mkdir -p coverage && npx c8 report --reporter=text-lcov > coverage/lcov.info
+ # - name: Coverage
+ # uses: coverallsapp/github-action@main
+ # with:
+ # github-token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -0,0 +1,36 @@
+############################
+# npm
+############################
+node_modules
+npm-debug.log
+.node_history
+yarn.lock
+package-lock.json
+
+############################
+# tmp, editor & OS files
+############################
+.tmp
+*.swo
+*.swp
+*.swn
+*.swm
+.DS_Store
+*#
+*~
+.idea
+*sublime*
+nbproject
+
+############################
+# Tests
+############################
+testApp
+coverage
+.nyc_output
+
+############################
+# Other
+############################
+.env
+.envrc

.npmrc

@@ -0,0 +1,13 @@
+audit=false
+enable-pre-post-scripts=true
+fund=false
+package-lock=false
+prefer-dedupe=true
+prefer-offline=true
+save-prefix=~
+save=false
+strict-peer-dependencies=false
+unsafe-perm=true
+loglevel=error
+shamefully-hoist=true
+resolution-mode=highest

LICENSE.md

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright © 2023 Kiko Beats <[email protected]> (kikobeats.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -0,0 +1,38 @@
+# top-crawler-agents
+
+![Last version](https://img.shields.io/github/tag/Kikobeats/top-crawler-agents.svg?style=flat-square)
+[![NPM Status](https://img.shields.io/npm/dm/top-crawler-agents.svg?style=flat-square)](https://www.npmjs.org/package/top-crawler-agents)
+
+> A list of common crawler user agents useful for retrieving metadata from links<br>
+> derivated from [crawler-user-agents](https://github.com/monperrus/crawler-user-agents).
+
+Some websites (such as [Twitter](https://twitter.com/Kikobeats/status/1687844145019092993)) only return rich HTML metadata if you are identified as popular crawler (like Slack, or WhatsApp).
+
+The list is derivated from [crawler-user-agents](https://github.com/monperrus/crawler-user-agents) after applying a [script](/scripts/generate.js), so it's always up-to-date to latest changes.
+
+## Install
+
+```bash
+$ npm install top-crawler-agents --save
+```
+
+## Usage
+
+```js
+const uniqueRandomArray = require('unique-random-array')
+const userAgents = require('top-crawler-agents')
+
+const randomUserAgent = uniqueRandomArray(userAgents)
+
+console.log(randomUserAgent())
+// => 'Slackbot-LinkExpanding (+https://api.slack.com/robots)'
+```
+
+## Related
+
+- [top-user-agents](https://github.com/Kikobeats/top-user-agents) – A list of most common User Agent used on Internet.
+
+**top-crawler-agents** © [Kiko Beats](https://kikobeats.com), released under the [MIT](https://github.com/Kikobeats/top-crawler-agents/blob/master/LICENSE.md) License.<br>
+Authored and maintained by [Kiko Beats](https://kikobeats.com) with help from [contributors](https://github.com/Kikobeats/top-crawler-agents/contributors).
+
+> [kikobeats.com](https://kikobeats.com) · GitHub [Kiko Beats](https://github.com/Kikobeats) · Twitter [@Kikobeats](https://twitter.com/Kikobeats)

index.js

@@ -0,0 +1 @@
+module.exports = require('./index.json')