json2yamlmd.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "SR0XORrEg04z"
      ],
      "authorship_tag": "ABX9TyOoOKGY4NvtGm5X7og+SKr1",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gabrielmacedoanac/ANAC/blob/main/json2yamlmd.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Gerar arquivos em .md"
      ],
      "metadata": {
        "id": "oR8WAGOfgvVS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install python-slugify\n",
        "!pip install pyyaml\n",
        "!pip install tabulate"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T5UhQ9jzkIBM",
        "outputId": "a96c1d92-72d5-469a-b561-8b44ceb1b43a"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: python-slugify in /usr/local/lib/python3.8/dist-packages (7.0.0)\n",
            "Requirement already satisfied: text-unidecode>=1.3 in /usr/local/lib/python3.8/dist-packages (from python-slugify) (1.3)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.8/dist-packages (6.0)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.8/dist-packages (0.8.10)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Montar o google drive\n",
        "from google.colab import drive\n",
        "drive.mount(\"/content/drive\", force_remount=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BBw-jFv2oKwY",
        "outputId": "a7ffcffb-3723-4b59-c974-91acd0730eed"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "id": "bg9An5MMjGh1"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import yaml\n",
        "from slugify import slugify\n",
        "\n",
        "# Carregar o dataframe\n",
        "df = pd.read_json(\"/content/drive/MyDrive/ANAC/regulamentos-anac-tags.json\")\n",
        "\n",
        "# Pegar apenas os 10 primeiros valores do dataframe para teste\n",
        "df = df[20000:]\n",
        "\n",
        "# Iterar sobre as linhas do dataframe\n",
        "for index, row in df.iterrows():\n",
        "  # YAML\n",
        "    # Criando um dicionário com os dados da linha atual\n",
        "    data = dict(row)\n",
        "    # Removendo chaves do dicionário que não são úteis para o YAML\n",
        "    data.pop('ementa')\n",
        "    data.pop('alterada')\n",
        "    data.pop('em_vigor')\n",
        "    data.pop('revogada')\n",
        "    data.pop('tornada_sem_efeito')\n",
        "    data.pop('outros')\n",
        "    # Convertendo o dicionário para YAML\n",
        "    yaml_data = yaml.dump(data, allow_unicode=True)\n",
        "\n",
        "  # Nome do arquivo\n",
        "    # Criar o nome do arquivo com base na segunda coluna ('norma')\n",
        "    nome_arquivo = row[1]\n",
        "    # Converter o nome do arquivo para minúsculas (slugify) e formato amigável para web\n",
        "    nome_arquivo = slugify(nome_arquivo)\n",
        "    nome_arquivo += \".md\"\n",
        "\n",
        "  # Esconder_toc\n",
        "    esconder_toc = \"hide: \\n- toc \\n\"\n",
        "\n",
        "  # Tipo_normatico\n",
        "    tipo_normatico = row[6]\n",
        "\n",
        "    filename = nome_arquivo\n",
        "                              # + '/' + nome_arquivo\n",
        "#    !mkdir /content/drive/MyDrive/my_folder\n",
        "\n",
        "  # Criar a tabela de markdown com os dados da linha atual\n",
        "    table_data = row[:-1].to_markdown(tablefmt=\"pipe\", index=True, headers=[\"Documento\"])\n",
        "\n",
        "  # Salvar os arquivos .md no Google Colab\n",
        "    with open(\"/content/drive/MyDrive/ANAC/json2yamlmd/\" + filename, \"w\") as file:\n",
        "        file.write(\"---\\n\" + yaml_data + esconder_toc + \" \\n---\\n\\n\" + table_data +\"\\n\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Salvar arquivo .zip\n",
        "import zipfile\n",
        "import os\n",
        "\n",
        "output_folder = '/content/drive/MyDrive/ANAC/json2yamlmd/'\n",
        "\n",
        "with zipfile.ZipFile('arquivos_md.zip', 'w') as zip_f:\n",
        "    for filename in os.listdir(output_folder):\n",
        "        if filename.endswith(\".md\"):\n",
        "            zip_f.write(os.path.join(output_folder, filename))"
      ],
      "metadata": {
        "id": "mUgYjjiYh0jJ"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Fazer download do arquivo .zip\n",
        "from google.colab import files\n",
        "files.download('arquivos_md.zip') "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "VQI5g-qkjtMj",
        "outputId": "f37fcfcc-bcf8-4042-ae97-f96c79bb594c"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_ddcb0540-dc47-45a5-9a52-702d76a211a4\", \"arquivos_md.zip\", 73500840)"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! rm -r /content/drive/MyDrive/ANAC/json2yamlmd/*.md"
      ],
      "metadata": {
        "id": "1KGpMild9dwf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Rascunhos"
      ],
      "metadata": {
        "id": "SR0XORrEg04z"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import yaml\n",
        "import re\n",
        "from slugify import slugify\n",
        "import zipfile\n",
        "import os\n",
        "\n",
        "# Carregando o dataframe\n",
        "df = pd.read_json(\"/content/drive/MyDrive/ANAC/regulamentos-anac-tags.json\")\n",
        "\n",
        "# Pegando apenas os 10 primeiros valores do dataframe\n",
        "# df = df[0:20000]\n",
        "\n",
        "# Configurando o yaml para o flavor do github\n",
        "yaml.Dumper.ignore_aliases = lambda self, data: True\n",
        "\n",
        "# Iterando sobre as linhas do dataframe\n",
        "for index, row in df.iterrows():\n",
        "    # Criando um dicionário com os dados da linha atual\n",
        "    data = dict(row)\n",
        "    # Convertendo o dicionário para YAML\n",
        "    yaml_data = yaml.dump(data, Dumper=yaml.SafeDumper)\n",
        "    # Criando o nome do arquivo com base na segunda coluna\n",
        "    filename = row[1]\n",
        "    # Convertendo o nome do arquivo para minúsculas\n",
        "    filename = slugify(filename)\n",
        "    filename += \".md\"\n",
        "    # Criando a tabela de markdown com os dados da linha atual\n",
        "    table_data = row.to_frame().to_html(header=False)\n",
        "    # Salvando o YAML e a tabela de markdown em um arquivo .md no Google Colab\n",
        "    with open(\"/content/drive/MyDrive/ANAC/csv2md/\" + filename, \"w\") as file:\n",
        "        file.write(yaml_data + \"\\n\\n\" + table_data)\n"
      ],
      "metadata": {
        "id": "668bPX1rhjoA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pyyaml\n",
        "!pip install tabulate"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vOEdg4W69W44",
        "outputId": "2bf9d947-1799-40d5-837f-fb1a4cc8e00c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.8/dist-packages (6.0)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.8/dist-packages (0.8.10)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ouRY3S_WccbR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "4yWWoHsJff0N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "4tsCO_u1ZZtM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "#import yaml\n",
        "import re\n",
        "from slugify import slugify\n",
        "import zipfile\n",
        "import os\n",
        "import yaml\n",
        "import sys\n",
        "\n",
        "\n",
        "# Carregando o dataframe\n",
        "df = pd.read_json(\"/content/drive/MyDrive/ANAC/regulamentos-anac-tags.json\")\n",
        "\n",
        "# Pegando apenas os 10 primeiros valores do dataframe\n",
        "df = df[0:2]\n",
        "\n",
        "# Substituindo os valores \"|\" por \", \"\n",
        "\n",
        "# Configurando o yaml para o flavor do github\n",
        "# yaml.Dumper.ignore_aliases = lambda self, data: True\n",
        "\n",
        "# Instanciando a biblioteca ruamel.yaml\n",
        "# yaml = YAML()\n",
        "\n",
        "# Iterando sobre as linhas do dataframe\n",
        "for index, row in df.iterrows():\n",
        "\n",
        "  # YAML\n",
        "    # Criando um dicionário com os dados da linha atual\n",
        "    data = dict(row)\n",
        "    # Removendo cahves do dicionário que não são úteis para o YAML\n",
        "    data.pop('ementa')\n",
        "    data.pop('alterada')\n",
        "    data.pop('em_vigor')\n",
        "    data.pop('revogada')\n",
        "    data.pop('tornada_sem_efeito')\n",
        "    data.pop('outros')\n",
        "    # Convertendo o dicionário para YAML\n",
        "    yaml_data = yaml.dump(data, allow_unicode=True)\n",
        "\n",
        "  # Nome do arquivo\n",
        "    # Criando o nome do arquivo com base na segunda coluna\n",
        "    filename = row[1]\n",
        "    # Convertendo o nome do arquivo para minúsculas\n",
        "    filename = slugify(filename)\n",
        "    filename += \".md\"\n",
        "\n",
        "  # Esconder_toc\n",
        "    esconder_toc = \"hide: \\n- toc \\n\"\n",
        "\n",
        "  # Tabela de conteúdo dentro do md\n",
        "    # Criando a tabela de markdown com os dados da linha atual\n",
        "    table_data = row.to_markdown(tablefmt=\"github\", index=True, headers=[\"Regulamento\"])\n",
        "    # .to_markdown(index=False, tablefmt=\"github\")\n",
        "\n",
        "  # Salvar o arquivo\n",
        "    # Salvando o YAML e a tabela de markdown em um arquivo .md no Google Colab\n",
        "    with open(\"/content/drive/MyDrive/ANAC/csv2md-yaml/\" + filename, \"w\") as file:\n",
        "        file.write(\"---\\n\" + yaml_data + esconder_toc + \" \\n---\\n\\n\" + table_data)\n"
      ],
      "metadata": {
        "id": "4q-_xygP8Bx4"
      },
      "execution_count": 116,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(data)\n",
        "yaml.Dumper.ignore_aliases = lambda self, data: True\n",
        "result = yaml.dump(data, allow_unicode=True)\n",
        "print(result)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WeiWMJ4a_6fO",
        "outputId": "2d8fa572-eaf9-496c-a207-a8f03ea9cea2"
      },
      "execution_count": 119,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'norma': 'Lei nº 12.648 de 17/05/2012', 'data': '17/05/2012', 'tipo_normatico': 'Lei', 'publicacao': 'DOU Nº 96, Seção 1, p.1, de 18/05/2012', 'anexos': 'https://www.anac.gov.br/assuntos/legislacao/legislacao-1/leis/lei-no-12-648-de-05-17-2012/@@display-file/arquivo_norma/Lei12648.pdf', 'tags': ['1.896', '12.462', '12.648', '5.862', '6.009', '7.920', '8.399', '9.825', 'altera', 'decreto', 'lei']}\n",
            "anexos: https://www.anac.gov.br/assuntos/legislacao/legislacao-1/leis/lei-no-12-648-de-05-17-2012/@@display-file/arquivo_norma/Lei12648.pdf\n",
            "data: 17/05/2012\n",
            "norma: Lei nº 12.648 de 17/05/2012\n",
            "publicacao: DOU Nº 96, Seção 1, p.1, de 18/05/2012\n",
            "tags:\n",
            "- '1.896'\n",
            "- '12.462'\n",
            "- '12.648'\n",
            "- '5.862'\n",
            "- '6.009'\n",
            "- '7.920'\n",
            "- '8.399'\n",
            "- '9.825'\n",
            "- altera\n",
            "- decreto\n",
            "- lei\n",
            "tipo_normatico: Lei\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "output_folder = '/content/drive/MyDrive/ANAC/csv2md-yaml/'\n",
        "\n",
        "with zipfile.ZipFile('arquivos_md2.zip', 'w') as zip_f:\n",
        "    for filename in os.listdir(output_folder):\n",
        "        if filename.endswith(\".md\"):\n",
        "            zip_f.write(os.path.join(output_folder, filename))"
      ],
      "metadata": {
        "id": "x6DtKkcjj6TM"
      },
      "execution_count": 120,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import shutil\n",
        "\n",
        "output_folder = '/content/drive/MyDrive/ANAC/csv2md-yaml/'\n",
        "\n",
        "# Criar um arquivo .zip com os arquivos .md\n",
        "shutil.make_archive(\"arquivos_md\", \"zip\", output_folder)\n",
        "\n",
        "# Caminho para o arquivo .zip\n",
        "zip_path = os.path.join(output_folder, \"arquivos_md.zip\")\n"
      ],
      "metadata": {
        "id": "7p9RIUBk1Z95"
      },
      "execution_count": 121,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "files.download('arquivos_md.zip') \n"
      ],
      "metadata": {
        "id": "hh3bF1-0kDY6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "files.download('arquivos_md2.zip') \n"
      ],
      "metadata": {
        "id": "OR3R2B3f3ntl"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}