text:
from bin.addons.arg_manager import arg_manager

def test_arg_manager_empty():
    argv = ["burn"]
    assert arg_manager(argv) == 0

def test_arg_manager_help():
    name = "burn"
    argv = ["-h", "--help", "help"]

    for i in range(0, 3):
        assert arg_manager([name, argv[i]]) == 0

def test_arg_manager_correct():
    argv = ["burn", "--target", "https://google.com",
            "--method", "http", "--time", "10"]

    assert arg_manager(argv) == {
        "threads": 3,
        "target": "https://google.com",
        "time": "10",
        "method": "http",
        "root": True
    }

repo_language: Python


text:
from bin.addons.ip_tools import GetTargetAddress
import re

def verify_ip(ip):
    return re.match(r"^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$", ip)

def test_get_target_addr_out_method():
    target = "https://google.com"
    method = "outmet"

    assert GetTargetAddress(target, method) == target

def test_get_target_addr_pod():
    https_target = "https://github.com"
    http_target = "http://github.com"
    url_target = "github.com"
    port_target = "80"
    ip_target = "140.82.113.3"

    method = "POD"

    ip, port = GetTargetAddress(https_target, method)
    assert verify_ip(ip) and port == 80

    ip, port = GetTargetAddress(http_target, method)
    assert verify_ip(ip) and port == 80

    url, port = GetTargetAddress(url_target + ":" + port_target, method)
    assert url == url_target and port == int(port_target)

    ip, port = GetTargetAddress(ip_target + ":" + port_target, method)
    assert verify_ip(ip) and port == int(port_target)

def test_get_target_addr_syn():
    https_target = "https://github.com"
    http_target = "http://github.com"
    url_target = "github.com"
    port_target = "80"
    ip_target = "140.82.113.3"

    method = "syn"

    ip, port = GetTargetAddress(https_target, method)
    assert verify_ip(ip) and port == 80

    ip, port = GetTargetAddress(http_target, method)
    assert verify_ip(ip) and port == 80

    url, port = GetTargetAddress(url_target + ":" + port_target, method)
    assert url == url_target and port == int(port_target)

    ip, port = GetTargetAddress(ip_target + ":" + port_target, method)
    assert verify_ip(ip) and port == int(port_target)

def test_get_target_addr_slowloris():
    https_target = "https://github.com"
    http_target = "http://github.com"
    url_target = "github.com"
    port_target = "80"
    ip_target = "140.82.113.3"

    method = "slowloris"

    ip, port = GetTargetAddress(https_target, method)
    assert verify_ip(ip) and port == 80

    ip, port = GetTargetAddress(http_target, method)
    assert verify_ip(ip) and port == 80

    url, port = GetTargetAddress(url_target + ":" + port_target, method)
    assert url == url_target and port == int(port_target)

    ip, port = GetTargetAddress(ip_target + ":" + port_target, method)
    assert verify_ip(ip) and port == int(port_target)

def test_get_target_addr_http():
    https_target = "https://github.com"
    http_target = "http://github.com"
    url_target = "github.com"
    port_target = "80"
    ip_target = "140.82.113.3"

    method = "http"

    ip, port = GetTargetAddress(https_target, method)
    assert verify_ip(ip) and port == 80

    ip, port = GetTargetAddress(http_target, method)
    assert verify_ip(ip) and port == 80

    url, port = GetTargetAddress(url_target + ":" + port_target, method)
    assert url == url_target and port == int(port_target)

    ip, port = GetTargetAddress(ip_target + ":" + port_target, method)
    assert verify_ip(ip) and port == int(port_target)

def test_get_target_addr_icmp():
    https_target = "https://github.com"
    http_target = "http://github.com"
    url_target = "github.com"
    port_target = "80"
    ip_target = "140.82.113.3"

    method = "icmp"

    ip, port = GetTargetAddress(https_target, method)
    assert verify_ip(ip) and port == 80

    ip, port = GetTargetAddress(http_target, method)
    assert verify_ip(ip) and port == 80

    url, port = GetTargetAddress(url_target + ":" + port_target, method)
    assert url == url_target and port == int(port_target)

    ip, port = GetTargetAddress(ip_target + ":" + port_target, method)
    assert verify_ip(ip) and port == int(port_target)

def test_get_target_addr_memcached():
    https_target = "https://github.com"
    http_target = "http://github.com"
    url_target = "github.com"
    port_target = "80"
    ip_target = "140.82.113.3"

    method = "memcached"

    ip, port = GetTargetAddress(https_target, method)
    assert verify_ip(ip) and port == 80

    ip, port = GetTargetAddress(http_target, method)
    assert verify_ip(ip) and port == 80

    url, port = GetTargetAddress(url_target + ":" + port_target, method)
    assert url == url_target and port == int(port_target)

    ip, port = GetTargetAddress(ip_target + ":" + port_target, method)
    assert verify_ip(ip) and port == int(port_target)

def test_get_target_addr_ntp():
    https_target = "https://github.com"
    http_target = "http://github.com"
    url_target = "github.com"
    port_target = "80"
    ip_target = "140.82.113.3"

    method = "ntp"

    ip, port = GetTargetAddress(https_target, method)
    assert verify_ip(ip) and port == 80

    ip, port = GetTargetAddress(http_target, method)
    assert verify_ip(ip) and port == 80

    url, port = GetTargetAddress(url_target + ":" + port_target, method)
    assert url == url_target and port == int(port_target)

    ip, port = GetTargetAddress(ip_target + ":" + port_target, method)
    assert verify_ip(ip) and port == int(port_target)

def test_get_target_addr_udp():
    https_target = "https://github.com"
    http_target = "http://github.com"
    url_target = "github.com"
    port_target = "80"
    ip_target = "140.82.113.3"

    method = "udp"

    ip, port = GetTargetAddress(https_target, method)
    assert verify_ip(ip) and port == 80

    ip, port = GetTargetAddress(http_target, method)
    assert verify_ip(ip) and port == 80

    url, port = GetTargetAddress(url_target + ":" + port_target, method)
    assert url == url_target and port == int(port_target)

    ip, port = GetTargetAddress(ip_target + ":" + port_target, method)
    assert verify_ip(ip) and port == int(port_target)

def test_get_target_addr_udp():
    https_target = "https://github.com"
    http_target = "http://github.com"
    url_target = "github.com"
    port_target = "80"
    ip_target = "140.82.113.3"

    method = "ssdp"

    ip, port = GetTargetAddress(https_target, method)
    assert verify_ip(ip) and port == 80

    ip, port = GetTargetAddress(http_target, method)
    assert verify_ip(ip) and port == 80

    url, port = GetTargetAddress(url_target + ":" + port_target, method)
    assert url == url_target and port == int(port_target)

    ip, port = GetTargetAddress(ip_target + ":" + port_target, method)
    assert verify_ip(ip) and port == int(port_target)

def test_get_target_addr_armageddos():
    https_target = "https://github.com"
    http_target = "http://github.com"
    url_target = "github.com"
    port_target = "80"
    ip_target = "140.82.113.3"

    method = "armageddos"

    ip, port = GetTargetAddress(https_target, method)
    assert verify_ip(ip) and port == 80

    ip, port = GetTargetAddress(http_target, method)
    assert verify_ip(ip) and port == 80

    url, port = GetTargetAddress(url_target + ":" + port_target, method)
    assert url == url_target and port == int(port_target)

    ip, port = GetTargetAddress(ip_target + ":" + port_target, method)
    assert verify_ip(ip) and port == int(port_target)


text:
from bin.addons.utils import random_IP
import re

def test_random_ip():
    ip = random_IP()
    assert re.match(r"^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$", ip)


text:
from bin.addons.utils import root_privileges

def test_root_privileges():
    assert root_privileges() == False or True


text:
 From [pixiv Inc. Service Master Terms of Use](https://policies.pixiv.net/en.html):
>
> ## Master Terms of Use
> ### Article 14. Prohibited Conduct
> When using the Services, User must not engage in the conduct set forth in the following items.
>
> 1. Using the Services to display, sell, purchase, register, or engage in other transactions involving the products set forth below:
>
>    1. Products that contain uncorrected representations of exposed sexual organs or explicit images of sexual intercourse;
>
> ## pixivFANBOX
> ### Article 7. Posting
> Creators may not publish posts that fall under any of the following. If the Company determines that a post falls under any one of the following, the Company is entitled to request that the registered Creator revise the post or make the post private in whole or in part, and depending on the circumstances, the Company may cancel the Creator registration of the relevant Creator.
>
> 1. Obscene or excessively deviant sexual expression;
>    ...
>
<br/>

From [Pixiv Guidelines](https://www.pixiv.net/terms/?page=guideline&lang=en):
>
> ## About censorship (mosaics/pixelation)
>
> * Generally speaking, the censorship process assumes that any inappropriate parts shall be hidden based on visual judgment rather than on the technical nature of the image.
> * Censorship shall be efficiently done until it has achieved a level where no programs can reverse the edit.
> * The range should include the outline.
> * Vague mosaics/pixelation where details are made clear by the transparency of the concealment shall not be considered as censorship.
> * The following are guidelines for using mosaics/pixelation:
>
>   * Mosaics should use squares at least 4x4 pixels in size. If the said image is more than 400 pixels long, the dimensions of the mosaic squares should be 1/100 the length of the whole image.
> * Censorship using solid colors should conceal all inappropriate parts of the image. However, exceptions can be made if concealing only a part of the image leads to an acceptable result.
> * Censorship using white liquids and other patterns or painting techniques which goal is censorship should conceal all inappropriate parts of the image. However, exceptions can be made if concealing only a part of the image leads to an acceptable result.
>
> ## Examples of expressions that require the use of mosaics/pixelation
>
> * Genitals or areas reminiscent of genitals
> * The union or joining of the genitals and the insertion of foreign objects
> * The union or joining of anuses and the insertion of foreign objects
> * Areas that are unnecessarily detailed due to being exposed by cuts,

provided by [Composingliger](https://twitter.com/composingliger)

repo_name: Qwaxi-Lixard/yes-porn-allowed

mime_type: text/plain


text:
---
name: Add Porn Allowed Site
about: Template for adding new site to Porn Allowed List.
title: Add "[SITE]" to Yes Porn Allow list
labels: documentation
assignees: ''

---

**What is the site you want to add?**

**What kind of service are they?**

**How does the site tend to treat NSFW content?**

**Do you know of any risk to the site?**

**Any comment you want to add about the site?**


text:
# Contributor Covenant Code of Conduct

## Our Pledge

We as members, contributors, and leaders pledge to make participation in our
community a harassment-free experience for everyone, regardless of age, body
size, visible or invisible disability, ethnicity, sex characteristics, gender
identity and expression, level of experience, education, socio-economic status,
nationality, personal appearance, race, religion, or sexual identity
and orientation.

We pledge to act and interact in ways that contribute to an open, welcoming,
diverse, inclusive, and healthy community.

## Our Standards

Examples of behavior that contributes to a positive environment for our
community include:

* Demonstrating empathy and kindness toward other people
* Being respectful of differing opinions, viewpoints, and experiences
* Giving and gracefully accepting constructive feedback
* Accepting responsibility and apologizing to those affected by our mistakes,
  and learning from the experience
* Focusing on what is best not just for us as individuals, but for the
  overall community

Examples of unacceptable behavior include:

* The use of sexualized language or imagery, and sexual attention or
  advances of any kind
* Trolling, insulting or derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or email
  address, without their explicit permission
* Other conduct which could reasonably be considered inappropriate in a
  professional setting

## Enforcement Responsibilities

Community leaders are responsible for clarifying and enforcing our standards of
acceptable behavior and will take appropriate and fair corrective action in
response to any behavior that they deem inappropriate, threatening, offensive,
or harmful.

Community leaders have the right and responsibility to remove, edit, or reject
comments, commits, code, wiki edits, issues, and other contributions that are
not aligned to this Code of Conduct, and will communicate reasons for moderation
decisions when appropriate.

## Scope

This Code of Conduct applies within all community spaces, and also applies when
an individual is officially representing the community in public spaces.
Examples of representing our community include using an official e-mail address,
posting via an official social media account, or acting as an appointed
representative at an online or offline event.

## Enforcement

Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
me@juliaferraioli.com.
All complaints will be reviewed and investigated promptly and fairly.

All community leaders are obligated to respect the privacy and security of the
reporter of any incident.

## Enforcement Guidelines

Community leaders will follow these Community Impact Guidelines in determining
the consequences for any action they deem in violation of this Code of Conduct:

### 1. Correction

**Community Impact**: Use of inappropriate language or other behavior deemed
unprofessional or unwelcome in the community.

**Consequence**: A private, written warning from community leaders, providing
clarity around the nature of the violation and an explanation of why the
behavior was inappropriate. A public apology may be requested.

### 2. Warning

**Community Impact**: A violation through a single incident or series
of actions.

**Consequence**: A warning with consequences for continued behavior. No
interaction with the people involved, including unsolicited interaction with
those enforcing the Code of Conduct, for a specified period of time. This
includes avoiding interactions in community spaces as well as external channels
like social media. Violating these terms may lead to a temporary or
permanent ban.

### 3. Temporary Ban

**Community Impact**: A serious violation of community standards, including
sustained inappropriate behavior.

**Consequence**: A temporary ban from any sort of interaction or public
communication with the community for a specified period of time. No public or
private interaction with the people involved, including unsolicited interaction
with those enforcing the Code of Conduct, is allowed during this period.
Violating these terms may lead to a permanent ban.

### 4. Permanent Ban

**Community Impact**: Demonstrating a pattern of violation of community
standards, including sustained inappropriate behavior,  harassment of an
individual, or aggression toward or disparagement of classes of individuals.

**Consequence**: A permanent ban from any sort of public interaction within
the community.

## Attribution

This Code of Conduct is adapted from the [Contributor Covenant][homepage],
version 2.0, available at
https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.

Community Impact Guidelines were inspired by [Mozilla's code of conduct
enforcement ladder](https://github.com/mozilla/diversity).

[homepage]: https://www.contributor-covenant.org

For answers to common questions about this code of conduct, see the FAQ at
https://www.contributor-covenant.org/faq. Translations are available at
https://www.contributor-covenant.org/translations.

repo_name: juliaferraioli/opensource-timeline


text:
# Contributing to the open source timeline

Please send us your contributions!

## Adding a new event

_Before adding a new event, make sure that it is not already in the timeline._

When adding a new event, gather the information requested by the [schema](README.md#schema). You may omit the optional elements, but take care to keep the placeholder commas in the row. To validate your addition, you can use [csvlint](https://csvlint.io/) along with the [schema.json](schema.json) file (this is enforced at pull request time, but validating it first prevents a lot of back and forth on PRs).

⚠ _When running `csvlint`, you will see the warning `start: inconsistent_values`; this is safe to ignore._

You do not need to create an issue to add an event, but please make sure your sources are trustworthy.

If you are uncertain if an event should be included in the timeline, please open an issue to discuss it.

## Submitting pull requests

We follow the [standard GitHub fork-change-pull request model](https://docs.github.com/en/get-started/quickstart/contributing-to-projects). Make your changes in a fork of this repository and submit your changes in a pull request.

We're not fussy about the pull request title. Just keep it simple, such as:

> Add new event for the release of `git`

or

> Update `git` entry tags

We may ask you to make stylistic or grammatical changes in the interest of clarity and consistency.

After a PR is merged, the timeline will be automatically sorted by the `start` field.

## Note

**An opened pull request does not mean that we will merge it into the timeline.**

There are events that we may consider irrelevant or too minor to include in the timeline. There aren't any hard and fast rules for this; it is up to the maintainers to make this judgement call.


text:
# Open source timeline

[![All Contributors](https://img.shields.io/badge/all_contributors-1-orange.svg?style=flat-square)](#contributors-)

The open source timeline aims to collect events and periods within open source history in a machine-readable format (comma separated value).

## Schema

Each entry in the timeline should have the following information:

- `start`, date in the format YYYY-MM-DD, YYYY-MM, or YYYY
- `end`, date in the format YYYY-MM-DD, YYYY-MM, or YYYY (optional; primarily used for periods)
- `title`, ideally a short string
- `description`, 1-3 sentences with more information
- `eventType`, using one of the following strings:
  - `release`, for events such as UNIX's fourth edition which was rewritten in `C` or the introduction of `git`
  - `vulnerability`, for events such as Heartbleed or Log4shell
  - `milestone`, for events such as Linux breaking 3% of market share for desktop in 2020 or the founding of the Open Source Initiative (OSI)
  - `other`, for events that don't fit into any of the above categories; please add tags if you use this `eventType`
- `eventImportance`, using one of the following strings:
  - `note`, for events that are more commentary or tangentially related to open source events
  - `low`, for minor events
  - `medium`, for normal-ish level events
  - `high`, for events that had a major impact on open source
  - `critical`, for events that had an impact beyond the open source ecosystem
- `referenceURI`, a link that acts as a citation for the event
- `imageURI`, a link to an image that represents the event (optional)
- `mediaURIs`, a list of links that provide additional context (optional)
- `tags`, a list of keywords that add context (optional)

The official schema is captured in [schema.json](schema.json), which is enforced when adding new entries to the timeline.

## Contributing

Please see the instructions in [CONTRIBUTING.md](CONTRIBUTING.md).

## Maintainer(s)

This timeline is maintained by [julia ferraioli](https://github.com/juliaferraioli) who is perpetually oversubscribed. Patience is appreciated!

## Contributors

Shane Curcuru
🔬

## License

This repository is licensed under [CC0 1.0 Universal](LICENSE).


text: * @juliaferraioli


text:
name: sort

on:
  push:
    branches: [ "main" ]
    paths: ["timeline.csv"]
  workflow_dispatch:

jobs:
  sort_timeline:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
      - name: Sort timeline according to start date
        run: |
          (head -n 1 timeline.csv && tail -n +2 timeline.csv | sort -k 1) > tmp
          mv tmp timeline.csv
      - name: Commit sorted timeline
        run: |
          if git diff --quiet timeline.csv; then
            echo "File is already sorted"
            exit 0
          fi
          git config user.name 'julia ferraioli'
          git config user.email 'me@juliaferraioli.com'
          git commit -am "Generated commit to automatically sort the timeline upon push"
          git push


text:
name: lint

on:
  pull_request:
    branches: [ "main" ]
    paths: ["timeline.csv"]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

jobs:
  # This workflow contains a single job called "build"
  basic_linting:
    # The type of runner that the job will run on
    runs-on: ubuntu-latest

    # Steps represent a sequence of tasks that will be executed as part of the job
    steps:
      - uses: actions/checkout@v3
      - uses: ruby/setup-ruby@v1
        with:
          ruby-version: '2.7'
      - name: Install csvlint
        run: gem install csvlint
      - name: Perform basic csv linting
        run: csvlint timeline.csv --schema=schema.json


text:
jobs:
- job: Build_and_Test
  displayName: Build and Test
  condition: succeeded()
  pool:
    name: Hosted Ubuntu 1604
  strategy:
    matrix:
      Python36:
        python.version: '3.6'
      Python37:
        python.version: '3.7'
    maxParallel: 3

  steps:
  - task: UsePythonVersion@0
    displayName: 'Use Python $(python.version)'
    inputs:
      versionSpec: '$(python.version)'

  - script: python -m pip install --upgrade pip
    displayName: 'Upgrade pip'

  - script: pip install -r requirements.txt
    displayName: 'Install requirements'

  - script: |
      flake8 --max-line-length=120 *.py hooks/ tests/
    displayName: 'Run lint (flake8) tests'

  - script: |
      pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov --cov-report=xml --cov-report=html
    displayName: pytest

  - task: PublishTestResults@2
    displayName: 'Publish Test Results **/test-results.xml'
    inputs:
      testResultsFiles: '**/test-results.xml'
      testRunTitle: 'Python $(python.version)'

  - task: PublishCodeCoverageResults@1
    inputs:
      codeCoverageTool: Cobertura
      summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
      reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov'

repo_name: equinor/data-science-template

repo_language: Jupyter Notebook


text:
cookiecutter
flake8
pytest
pytest-cookies
pytest-cov


text:
[![Build Status](https://dev.azure.com/mhew/data-science-template/_apis/build/status/data-science-template?branchName=master)](https://dev.azure.com/mhew/data-science-template/_build/latest?definitionId=15&branchName=master)

# Data Science Template
This is a starter template for data science projects in Equinor, although it may also be useful for others. It contains many of the essential artifacts that you will need and presents a number of best practices including code setup, samples, MLOps using Azure, a standard document to guide and gather information relating to the data science process and more.

As it is impossible to create a single template that will meet every projects needs, this example should be considered
a starting point and changed based upon the working and evolution of your project.

Before working with the contents of this template or Data Science projects in general it is recommended to familiarise yourself with the Equinor [Data Science Technical Standards](https://wiki.statoil.no/wiki/index.php/Statoil_Data_Science_Technical_Standards) (Currently Equinor internal only)

## Getting Started With This Template
This template is provided as a [Cookiecutter template](http://cookiecutter.readthedocs.org/en/latest/installation.html) so you
can quickly create an instance customised for your project. An assumption is that you have a working python installation.

To get running, first install the latest Cookiecutter if you haven't installed it yet (this requires
Cookiecutter 1.4.0 or higher):

    pip install -U cookiecutter

### Create project
Then generate a new project for your own use based upon the template, answering the questions to customise the generated
project:

    cookiecutter https://github.com/equinor/data-science-template.git

The values you are prompted for are:

| Value                   | Description |
| :---                    | --- |
| project_name            | A name for your project. Used mostly within documentation |
| project_description     | A description to include in the README.md |
| repo_name               | The name of the github repository where the project will be held |
| conda_name              | The name of the conda environment to use |
| package_name            | A name for the generated python package. |
| mlops_name              | Default name for Azure ML. |
| mlops_compute_name      | Default Azure ML compute cluster name to use. |
| author                  | The main author of the solution. Included in the setup.py file |
| open_source_license     | What type of open source license the project will be released under |
| devops_organisation     | An Azure DevOps organisation. Leave blank if you aren't using Azure DevOps |

If you are uncertain about what to enter for any value then just accept the defaults. You can always change the generated project later.

*Getting problems? You can always download this repository using the download button above and reference the local copy e.g. cookiecutter c:\Downloads\data-science-template, however ideally fix any git proxy or other issues that are causing problems.*

You are now ready to get started, however you should first create a new github repository for your new project and add your
project using the following commands (substitute myproject with the name of your project and REMOTE-REPOSITORY-URL
with the remote repository url).

    cd myproject
    git init
    git add .
    git commit -m "Initial commit"
    git remote add origin REMOTE-REPOSITORY-URL
    git remote -v
    git push origin master

### Continuous Integration
Continuous Integration (CI) increase quality by building, running tests and performing other validation whenever
code is committed. The template contains a build pipeline for Azure DevOps, however requires a couple of manual
steps to setup:

* Log in to http://dev.azure.com and browse to, or create an organisation & project. The project name should be the same as your github repository name.
* Under *Pipelines -> Builds select* *New Pipeline*
* Select github and then your repository. Login / grant any permissions as prompted
* In the review pane click *run*

You are now setup for CI and automated test / building. You should verify the badge link in this README corresponds
with your DevOps project, and as a further step might setup any release pipelines for automated deployment.

At this stage the build pipeline doesn't include MLOps steps, although these can be added based uon your needs.

### Finally

* Update the project readme file with additional project specific details including setup, configuration and usage.
* The docs\process_documentation.md file should be completed phase by phase, and each phase result shall be submitted for review and approval before the project moves on to the next phase. This is to assist with the gathering of essential information required to deliver a correct and robust solution. The git respoitory shall be added to the script that populates the [knowledge repository](https://git.statoil.no/DataScience/projects) to ease future knowledge sharing.

## Generated Project Contents
Depending upon the selected options when creating the project, the generated structure will look similar to the below:

```
├── .gitignore               <- Files that should be ignored by git. Add seperate .gitignore files in sub folders if
│                               needed
├── conda_env.yml            <- Conda environment definition for ensuring consistent setup across environments
├── LICENSE
├── README.md                <- The top-level README for developers using this project.
├── requirements.txt         <- The requirements file for reproducing the analysis environment, e.g.
│                               generated with `pip freeze > requirements.txt`. Might not be needed if using conda.
├── setup.py                 <- Metadata about your project for easy distribution.
│
├── data
│   ├── interim_[desc]       <- Interim files - give these folders whatever name makes sense.
│   ├── processed            <- The final, canonical data sets for modeling.
│   ├── raw                  <- The original, immutable data dump.
│   ├── temp                 <- Temporary files.
│   └── training             <- Files relating to the training process
│
├── docs                     <- Documentation
│   ├── data_science_code_of_conduct.md  <- Code of conduct.
│   ├── process_documentation.md         <- Standard template for documenting process and decisions.
│   └── writeup              <- Sphinx project for project writeup including auto generated API.
│      ├── conf.py           <- Sphinx configurtation file.
│      ├── index.rst         <- Start page.
│      ├── make.bat          <- For generating documentation (Windows)
│      └── Makefikle         <- For generating documentation (make)
│
├── examples                 <- Add folders as needed e.g. examples, eda, use case
│
├── extras                   <- Miscellaneous extras.
│   └── add_explorer_context_shortcuts.reg    <- Adds additional Windows Explorer context menus for starting jupyter.
│
├── notebooks                <- Notebooks for analysis and testing
│   ├── eda                  <- Notebooks for EDA
│   │   └── example.ipynb    <- Example python notebook
│   ├── features             <- Notebooks for generating and analysing features (1 per feature)
│   ├── modelling            <- Notebooks for modelling
│   └── preprocessing        <- Notebooks for Preprocessing
│
├── scripts                  <- Standalone scripts
│   ├── deploy               <- MLOps scripts for deployment (WIP)
│   │   └── score.py         <- Scoring script
│   ├── train                <- MLOps scripts for training
│   │   ├── submit-train.py  <- Script for submitting a training run to Azure ML Service
│   │   ├── submit-train-local.py <- Script for local training using Azure ML
│   │   └── train.py         <- Example training script using the iris dataset
│   ├── example.py           <- Example sctipt
│   └── MLOps.ipynb          <- End to end MLOps example (To be refactored into the above)
│
├── src                      <- Code for use in this project.
│   └── examplepackage       <- Example python package - place shared code in such a package
│       ├── __init__.py      <- Python package initialisation
│       ├── examplemodule.py <- Example module with functions and naming / commenting best practices
│       ├── features.py      <- Feature engineering functionality
│       ├── io.py            <- IO functionality
│       └── pipeline.py      <- Pipeline functionality
│
└── tests                    <- Test cases (named after module)
    ├── test_notebook.py     <- Example testing that Jupyter notebooks run without errors
    ├── examplepackage       <- examplepackage tests
        ├── examplemodule    <- examplemodule tests (1 file per method tested)
        ├── features         <- features tests
        ├── io               <- io tests
        └── pipeline         <- pipeline tests
```

## Contributing to This Template
Contributions to this template are greatly appreciated and encouraged.

To contribute an update simply:
* Submit an issue describing your proposed change to the repo in question.
* The repo owner will respond to your issue promptly.
* Fork the desired repo, develop and test your code changes.
* Check that your code follows the PEP8 guidelines (line lengths up to 120 are ok) and other general conventions within this document.
* Ensure that your code adheres to the existing style. Refer to the
   [Google Cloud Platform Samples Style Guide](
   https://github.com/GoogleCloudPlatform/Template/wiki/style.html) for the
   recommended coding standards for this organization.
* Ensure that as far as possible there are unit tests covering the functionality of any new code.
* Check that all existing unit tests still pass.
* Edit this document and the template README.md if needed to describe new files or other important information.
* Submit a pull request.

### Template development environment
To develop this template further you might want to setup a virtual environment

#### Setup using
```
cd data-science-template
python -m venv dst-env
```

#### Activate environment
Max / Linux
```
source dst-env/bin/activate
```

Windows
```
dst-env\Scripts\activate
```

#### Install Dependencies
```
pip install -r requirements.txt
```

#### Testing
To run the template tests, install pytest using pip or conda and then from the repository root run

    pytest tests

#### Linting
To verify that your code adheres to python standards run linting as shown below:

    flake8 --max-line-length=120 *.py hooks/ tests/

## Important Links
* https://wiki.statoil.no/wiki/index.php/Statoil_Data_Science_Technical_Standards - Data Science Technical Standards (Equinor Internal)
* https://dataplatformwiki.azurewebsites.net/doku.php - Data Platform wiki (Equinor internal)
* https://github.com/Statoil/data-science-shared - Shared Data Science Code Repository (Equinor internal)

## References
* https://github.com/Statoil/data-science-template/ - The master template for this project
* http://docs.python-guide.org/en/latest/writing/structure/
* https://github.com/Azure/Microsoft-TDSP
* https://drivendata.github.io/cookiecutter-data-science/
* https://github.com/audreyr/cookiecutter-pypackage

[//]: #
   [anaconda]: <https://www.continuum.io/downloads>


text:
# import datetime
import os
# import shutil
# from os.path import join

def replace_contents(filename: str, what: str, replacement: str) -> None:
    """
    Replace instances of a given string in a file

    Args:
        filename: The filename to replace within
        what: The text that should be matched
        replacement: The text that what should be replaced with
    """
    with open(filename) as fh:
        changelog = fh.read()
    with open(filename, 'w') as fh:
        fh.write(changelog.replace(what, replacement))

if __name__ == "__main__":
    # today = datetime.date.today()
    # replace_contents('LICENSE', '<YEAR>', today.strftime("%Y"))

    if '{{ cookiecutter.open_source_license }}' == "Not open source":
        os.remove('LICENSE')
        # shutil.rmtree('LICENSE')

    # Print out some information on setup and next steps
    print("""

Data Science Project '{{ cookiecutter.repo_name }}' created using the following
parameters:

{% for key, value in cookiecutter.items()|sort %}
    {{ "{0:26}".format(key + ":") }} {{ "{0!r}".format(value).strip("u") }}
{%- endfor %}

You are now ready to get started, however you should create a new github
repository for your new project and add your project using the following
commands (substitute REMOTE-REPOSITORY-URL with the remote repository url).

    cd {{ cookiecutter.repo_name }}
    git init
    git add --all
    git commit -m "Initial commit"
    git remote add origin REMOTE-REPOSITORY-URL
    git push -u origin master
""")


text:
name: {{cookiecutter.conda_name}}
dependencies:
  - pandas=0.23.4
  - pytest=5.3.1
  - pytest-cov=2.8.1
  - numpy=1.17.4
  - nbconvert=5.6.1
  - nbformat=4.4.0
  - pip:
    - azureml-sdk
    - joblib==0.14.1
    - matplotlib==3.1.2
    - scikit-learn==0.22.1


text:
jobs:
- job: Build_and_Test
  displayName: Build and Test
  condition: succeeded()
  pool:
    name: Hosted Ubuntu 1604
  strategy:
    matrix:
      Python36:
        python.version: '3.6'
      Python37:
        python.version: '3.7'
    maxParallel: 3

  steps:
  - task: UsePythonVersion@0
    displayName: 'Use Python $(python.version)'
    inputs:
      versionSpec: '$(python.version)'

  - script: python -m pip install --upgrade pip
    displayName: 'Upgrade pip'

  - script: pip install -r requirements.txt
    displayName: 'Install requirements'

  - script: |
      flake8 --max-line-length=120 *.py tests/ scripts/
    displayName: 'Run lint (flake8) tests'

  - script: |
      pytest tests --doctest-modules --junitxml=junit/test-results.xml --cov --cov-report=xml --cov-report=html
    displayName: pytest

  - task: PublishTestResults@2
    displayName: 'Publish Test Results **/test-results.xml'
    inputs:
      testResultsFiles: '**/test-results.xml'
      testRunTitle: 'Python $(python.version)'

  - task: PublishCodeCoverageResults@1
    inputs:
      codeCoverageTool: Cobertura
      summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
      reportDirectory: '$(System.DefaultWorkingDirectory)/**/htmlcov'


text:
flake8
m2r
nbformat
numpy
pandas
pytest
pytest-cookies
pytest-cov
sphinx


text:
import os
from setuptools import find_packages, setup

# Utility function to read the README file.
# Used for the long_description.  It's nice, because now 1) we have a top level
# README file and 2) it's easier to type in the README file than to put a raw
# string in below ...
def read(file_name):
    return open(os.path.join(os.path.dirname(__file__), file_name)).read()

{%- set license_classifiers = {
    'MIT': 'License :: OSI Approved :: MIT License',
    'LGPL3': 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)',
    'GPL3': 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)'
} %}

install_requires = [
    'numpy',
    'pandas',
    'pytest'
]

setup_requirements = [
    'pytest-runner',
    'better-setuptools-git-version'
]

test_requirements = [
    'pytest',
    'nbformat'
]

setup(
    author='{{cookiecutter.author}}',
    author_email="Name@equinor.com",
    classifiers=[
        "Development Status :: 3 - Alpha",
        "Topic :: Utilities",
{%- if cookiecutter.open_source_license in license_classifiers %}
        '{{ license_classifiers[cookiecutter.open_source_license] }}',
{%- endif %}
    ],

    name="{{cookiecutter.project_name}}",
    # version="0.0.1",
    version_config={
      "version_format": "{tag}.dev{sha}",
      "starting_version": "0.0.1"
    },
    description="{{cookiecutter.project_description}}",
    long_description=open('README.md').read(),
    packages=find_packages('src'),
    package_dir={'': 'src'},
    setup_requires=setup_requirements,
    test_suite='tests',
    tests_require=test_requirements,
    install_requires=install_requires
)


text:
{% if cookiecutter.devops_organisation != '' %}
[![Build Status](https://dev.azure.com/{{cookiecutter.devops_organisation}}/{{cookiecutter.repo_name}}/_apis/build/status/equinor.{{cookiecutter.repo_name}}?branchName=master)](https://dev.azure.com/{{cookiecutter.devops_organisation}}/{{cookiecutter.repo_name}}/_build/latest?definitionId=1&branchName=master)
{% endif %}

# {{cookiecutter.project_name}}

{{cookiecutter.project_description}}

## Setup
1. Install git and checkout the [git code repository]
2. Install [anaconda] python version 3.6+
3. Change working directory into the git code repository root
4. Create the self contained conda environment. In a terminal go to the git code repository root and enter the command:

   `conda env create --file conda_env.yml`

5. Any python modules under src need to be available to other scripts. This can be done in a couple of ways. You can
setup and install the python modules by executing the setup.py command below which will install the packages to the
conda environments site-packages folder but with a symlink to the src folder so modifications are reflected immediately.

   `python setup.py develop`

    As an alternative you may prefer to set the python path directly from the console, within notebooks, test scripts
    etc. From Pycharm you can also right click the src folder and select the _Mark Directory As | Source Root_ option.

6. .. Place your own project specific setup steps here e.g. copying data files ...

When distributing your module, you can create a Python egg with the command `python setup.py bdist_egg` and upload the egg.

NOTE: When working in the project notebooks from within the Equinor network, you may need to include the lines below if your proxy is not otherwise setup.

`os.environ['HTTP_PROXY']="http://www-proxy.statoil.no:80"`<br />
`os.environ['HTTPS_PROXY']="http://www-proxy.statoil.no:80"`

## Using the Python Conda environment

Once the Python Conda environment has been set up, you can

* Activate the environment using the following command in a terminal window:

  * Windows: `activate {{cookiecutter.conda_name}}`
  * Linux, OS X: `source activate {{cookiecutter.conda_name}}`
  * The __environment is activated per terminal session__, so you must activate it every time you open terminal.

* Deactivate the environment using the following command in a terminal window:

  * Windows: `deactivate {{cookiecutter.conda_name}}`
  * Linux, OS X: `source deactivate {{cookiecutter.conda_name}}`

* Delete the environment using the command (can't be undone):

  * `conda remove --name {{cookiecutter.conda_name}} --all`

## Initial File Structure

```
├── .gitignore               <- Files that should be ignored by git. Add seperate .gitignore files in sub folders if
│                               needed
├── conda_env.yml            <- Conda environment definition for ensuring consistent setup across environments
├── LICENSE
├── README.md                <- The top-level README for developers using this project.
├── requirements.txt         <- The requirements file for reproducing the analysis environment, e.g.
│                               generated with `pip freeze > requirements.txt`. Might not be needed if using conda.
├── setup.py                 <- Metadata about your project for easy distribution.
│
├── data
│   ├── interim_[desc]       <- Interim files - give these folders whatever name makes sense.
│   ├── processed            <- The final, canonical data sets for modeling.
│   ├── raw                  <- The original, immutable data dump.
│   ├── temp                 <- Temporary files.
│   └── training             <- Files relating to the training process
│
├── docs                     <- Documentation
│   ├── data_science_code_of_conduct.md  <- Code of conduct.
│   ├── process_documentation.md         <- Standard template for documenting process and decisions.
│   └── writeup              <- Sphinx project for project writeup including auto generated API.
│      ├── conf.py           <- Sphinx configurtation file.
│      ├── index.rst         <- Start page.
│      ├── make.bat          <- For generating documentation (Windows)
│      └── Makefikle         <- For generating documentation (make)
│
├── examples                 <- Add folders as needed e.g. examples, eda, use case
│
├── extras                   <- Miscellaneous extras.
│   └── add_explorer_context_shortcuts.reg    <- Adds additional Windows Explorer context menus for starting jupyter.
│
├── notebooks                <- Notebooks for analysis and testing
│   ├── eda                  <- Notebooks for EDA
│   │   └── example.ipynb    <- Example python notebook
│   ├── features             <- Notebooks for generating and analysing features (1 per feature)
│   ├── modelling            <- Notebooks for modelling
│   └── preprocessing        <- Notebooks for Preprocessing
│
├── scripts                  <- Standalone scripts
│   ├── deploy               <- MLOps scripts for deployment (WIP)
│   │   └── score.py         <- Scoring script
│   ├── train                <- MLOps scripts for training
│   │   ├── submit-train.py  <- Script for submitting a training run to Azure ML Service
│   │   ├── submit-train-local.py <- Script for local training using Azure ML
│   │   └── train.py         <- Example training script using the iris dataset
│   ├── example.py           <- Example sctipt
│   └── MLOps.ipynb          <- End to end MLOps example (To be refactored into the above)
│
├── src                      <- Code for use in this project.
│   └── {{cookiecutter.package_name}}       <- Example python package - place shared code in such a package
│       ├── __init__.py      <- Python package initialisation
│       ├── examplemodule.py <- Example module with functions and naming / commenting best practices
│       ├── features.py      <- Feature engineering functionality
│       ├── io.py            <- IO functionality
│       └── pipeline.py      <- Pipeline functionality
│
└── tests                    <- Test cases (named after module)
    ├── test_notebook.py     <- Example testing that Jupyter notebooks run without errors
    └── {{cookiecutter.package_name}}       <- {{cookiecutter.package_name}} tests
        ├── examplemodule    <- examplemodule tests (1 file per method tested)
        ├── features         <- features tests
        ├── io               <- io tests
        └── pipeline         <- pipeline tests
```

## MLOps
Starter scripts for MLOps with Azure ML Service are included as a part of this template in the scripts folder and may be
customised for your own purposes. Please browse the contents of the scripts folder for more details.

For model training, the provided setup allows for running locally without any dependency on Azure ML by running train.py
in the scripts/train folder directly. Alternatively you can submit local or remote runs using the submit scripts in the
same folder.

## Testing
Reproducability and the correct functioning of code are essential to avoid wasted time. If a code block is copied more
than once then it should be placed into a common script / module under src and unit tests added. The same applies for
any other non trivial code to ensure the correct functioning.

To run tests, install pytest using pip or conda (should have been setup already if you used the conda_env.yml file) and
then from the repository root run

```
pytest
```

## Automated Document Generation
A [sphinx](https://www.sphinx-doc.org/) project is provided under docs/writeup that will generate writeup that
also includes automatically generated API information for any packages. THe output can be created in multiple
formats including html and pdf. If you are using CI then this can be run automatically. To run
locally execute the following commands:

```
cd docs/writeup
make html
```

On Windows this will run the make.bat, a Makefile is also included for those using the 'make' command.

## Development Process
Contributions to this template are greatly appreciated and encouraged.

To contribute an update simply:
* Create a new branch / fork for your updates.
* Check that your code follows the PEP8 guidelines (line lengths up to 120 are ok) and other general conventions within this document.
* Ensure that as far as possible there are unit tests covering the functionality of any new code.
* Check that all existing unit tests still pass.
* Edit this document if needed to describe new files or other important information.
* Create a pull request.

## Important Links
* https://wiki.equinor.com/wiki/index.php/Statoil_Data_Science_Technical_Standards - Data Science Technical Standards (Equinor Internal)
* https://dataplatformwiki.azurewebsites.net/doku.php - Data Platform wiki (Equinor internal)
* https://github.com/equinor/data-science-shared - Shared Data Science Code Repository (Equinor internal)

## References
* https://github.com/equinor/data-science-template/ - The master template for this project
* http://docs.python-guide.org/en/latest/writing/structure/
* https://github.com/Azure/Microsoft-TDSP
* https://drivendata.github.io/cookiecutter-data-science/

[//]: #
   [anaconda]: <https://www.continuum.io/downloads>


text:
# Code of Conduct

## Authorship and validity
This code of conduct is derived from the “Data Science Code of Professional Conduct” written by the Data Science Association. The full version is available at [http://www.datascienceassn.org/code-of-conduct.html](http://www.datascienceassn.org/code-of-conduct.html). For definitions of the terminology in this code of conduct please consult the full version.

This code of conduct attempts to distill the fuller code to a more manageable level, a code which is easier to communicate whilst (hopefully) still retaining the spirit of the Data Association’s code of conduct. We have done this through highlighting what we believe to be the most important points for data science in Equinor. Points conspicuous in their absence should not be assumed to be invalid. The full code of conduct is embraced in its entirety.

The aim of this code of conduct is to help ensure the quality of Data Science carried out at Equinor is at least in accordance with the guidelines set out below. This includes the work of internal Data Scientists, other internal resources and external partners and vendors. Indeed, all data science which is delivered to Equinor.

Key tenets are competence, honesty, scientific integrity and repeatability.

The rules detailed here are intended to augment relevant laws and regulations which are valid in the local where the data science work is being delivered (e.g. the European Union’s General Data Protection Regulation). In the event of any perceived conflict between this code of conduct and local rules and regulations it is the local rules and regulations which take precedence.

## Success criteria and validation

* **Rule 0:** The data scientist shall, in collaboration with the client, develop success criteria for the purposes of establishing metrics by which the quality of the delivery can be assessed. Success criteria shall:
  * **a)**	As much as possible, be based on objective, industry accepted quantitative metrics such as precision and recall.
  * **b)**	Be agreed as early as possible in the data science delivery
  * **c)**	Remain unchanged throughout the delivery, unless it is agreed with the client that the success criteria are not appropriate
  * **d)**	Not be repeatedly adjusted in light of unfavourable evidence. When re-establishing success criteria (as of rule 0c), great care must be taken to ensure that this will change will support the quality of the delivery
  * **e)**	Where quantitative metrics cannot be agreed due to the nature of the problem, e.g. language models, anomaly detection with low rate of known anomalies, an honest assessment of the quality of the delivery is still required
  * **f)**	Success criteria should focus on generalisation of models and workflows.

## Client Communication

* **Rule 1:** A data scientist shall consult with the client about any real, perceived and potentially hidden risks in relying on data science results.

* **Rule 2:** A data scientist shall explain data science results to the extent reasonably necessary to permit the client to make informed decisions regarding the data science.
Confidential Information

* **Rule 3:** Confidential information is information that the data scientist creates, develops, receives, uses or learns during employment as a data scientist for a client, either working directly in-house as an employee of an organization or as an independent professional. It includes information that is not generally known by the public about the client, including client affiliates, employees, customers or other parties with whom the client has a relationship and who have an expectation of confidentiality. The data scientist has a professional duty to protect all confidential information, regardless of its form or format, from the time of its creation or receipt until its authorised disposal.

* **Rule 4:** A data scientist shall make reasonable efforts to prevent the inadvertent or unauthorized disclosure of, or unauthorized access to, information relating to the representation of a client, which means:
  * **a)** Not displaying, reviewing or discussing confidential information in public places, in the presence of third parties or that may be overheard;
  * **b)** Not e-mailing confidential information outside of the organization or professional practice to a personal e-mail account or otherwise removing confidential information from the client by removing hard copies or copying it to any form of recordable digital media device; and
  * **c)** Communicating confidential information only to client employees and authorized agents (such as legal professionals or external auditors) who have a legitimate business reason to know the information.

* **Rule 5:** A data scientist shall comply with client policies that apply to the acceptance, proper use and handling of confidential information, as well as any written agreements between the data scientist and the client relating to confidential information.

* **Rule 6:** A data scientist shall protect client confidential information after termination of work for the client.
* **Rule 7:** A data scientist shall return any and all confidential information in possession or control upon termination of the data scientist - client relationship and, if requested, execute an affidavit affirming compliance with obligations relating to confidential information.

## Data Science Evidence, Quality of Data and Quality of Evidence

* **Rule 8:** A data scientist shall inform the client of all data science results and material facts known to the data scientist that will enable the client to make informed decisions, whether or not the data science evidence is adverse.

* **Rule 9:** The data scientist understands that bad or uncertain data quality may compromise data science professional practice and may communicate a false reality or promote an illusion of understanding. The data scientist shall take reasonable measures to protect the client from relying and making decisions based on bad or uncertain data quality.

* **Rule 10:** The data scientist understands that evidence may be weak or strong or uncertain and shall take reasonable measures to protect the client from relying and making decisions based on weak or uncertain evidence.

* **Rule 11:** A data scientist shall not knowingly:
  * **a)** fail to use scientific methods in performing data science;
  * **b)** fail to convey the quality of evidence in a reasonable and understandable manner for the client;
  * **c)** claim weak or uncertain evidence is strong evidence;
  * **d)** misuse weak or uncertain evidence to communicate a false reality or promote an illusion of understanding;
  * **e)** fail to convey the quality of data in a reasonable and understandable manner for the client;
  * **f)** claim bad or uncertain data quality is good data quality;
  * **g)** misuse bad or uncertain data quality to communicate a false reality or promote an illusion of understanding;
  * **h)** engage in cherry-picking (pointing to individual cases or data that seem to confirm a particular position, while ignoring a significant portion of related cases or data that may contradict that position of data or data science evidence);
  * **i)** fail to attempt to replicate data science results;
  * **j)** fail to disclose that data science results could not be replicated;
  * **k)** misuse data science results to communicate a false reality or promote an illusion of understanding;
  * **l)** fail to disclose failed experiments or disconfirming evidence known to the data scientist to be directly adverse to the position of the client;
  * **m)** offer evidence that the data scientist knows to be false. If a data scientist questions the quality of data or evidence the data scientist must disclose this to the client. If a data scientist has offered material evidence and the data scientist comes to know of its falsity, the data scientist shall take reasonable remedial measures, including disclosure to the client. A data scientist may disclose and label evidence the data scientist reasonably believes is false.

* **Rule 12:** A data scientist shall use reasonable diligence when assigning value and meaning to the following concepts when conducting data science:
  * **a)** "Statistically Significant"
  * **b)** "Correlation"
  * **c)** "Spurious Correlation"
  * **d)** "Causation"

* **Rule 13:** A data scientist shall not present incomplete evidence as real data science evidence. A data scientist may present a theory constituting incomplete evidence but shall label and clearly communicate the use of incomplete evidence.

* **Rule 14:** A data scientist shall use the data science method which consists of the following steps:
  * **a)** Careful observations of data, data sets and relationships between data;
  * **b)** Deduction of meaning from the data and different data relationships;
  * **c)** Formation of hypotheses;
  * **d)** Experimental or observational testing of the validity of the hypotheses. To be termed scientific, a method of inquiry must be based on empirical and measurable evidence subject to specific principles of reasoning.


text:
# Data Science Process

## Introduction
This DS process standard takes the form of a template. Each DS project shall complete this template to make sure that DS project in Statoil can deliver value to business to the maximal extent.

The document is structured into 6 sections according to the 6 DS phases:
* Business understanding and problem definition
* Project planning
* Collecting and preparing data
* Modeling
* Evaluation
* Deployment and monitoring

For each phase, a set of tasks are identified. A group of questions are listed under each task. These questions are designed to guide DS teams to carry out DS projects in a standard way.

The DS team shall complete all the questions as a joint effort of the whole team. The answers for each phase shall be reviewed/approved by QA team (for example, DS VP, PM, DS discipline advisor, Peers, Stakeholders) before the project moves to the next phase (Table 1. shall contain the review/approval records for each phase). The questions answered in previous iterations shall be updated if needed, and the updated document shall be reviewed and approved.

DS project leader (DS team responsible) owns (is responsible for completing) this document.

Figure 1. shows the adopted data science process phases, and the document/quality control process is plotted in the middle. A more detailed document/quality control flowchart is illustrated in Figure 2.

The document shall be added to the script that populates the [knowledge repository](https://git.statoil.no/DataScience/projects) to ease future knowledge sharing.

* * *
**Table 1.** Data Science phase review/approval table

* * *

**Figure 1.** Data Science process diagram

_ _ _

**Figure 2.** Data Science process FlowChart

_ _ _

## 1 Business Understanding and Problem Definition

### 1.1 Connecting business
#####   1.1.1 Who is the sponsor?
##### 	1.1.2 What is the business organization chart?
##### 	1.1.3	What is the business goal/objective?
##### 	1.1.4	What is the business strategy?
##### 	1.1.5	What is the business process?
##### 	1.1.6	How is the business efficiency measured?
##### 	1.1.7	Who is the main contact person?
##### 	1.1.8	Who is the SME?
##### 	1.1.9	Who is the end user?

### 	1.2	Understanding business challenge
##### 	1.2.1	What is the business challenge/pain point/bottleneck?
##### 	1.2.2	What are the identified use cases?
##### 	1.2.3	Has the discovery workshop been executed, and what is the result?
##### 	1.2.4	What is the current solution / how might a theoretical human “expert” perform the task today?
##### 	1.2.5	If your human expert were to perform this task, how would you respond to them so they improved for the next time? For classification, do this for all four phases of a typical confusion matrix (e.g. true / false)
##### 	1.2.6	If a human were to perform this task, what assumptions would the user want them to make?
##### 	1.2.7	What are the constraints?

###	1.3	Identifying requirements
##### 	1.3.1	What are the functional requirements?
##### 	1.3.2	What are the identified end user UX requirements?
##### 	1.3.3	How should the potential solution fit into the business process?
##### 	1.3.4	What is the solution delivery deadline?

###	1.4	Discovering data
##### 	1.4.1	What data do you need ideally to solve the problem?
##### 	1.4.2	What data is available?
##### 	1.4.3	Have you use Statoil data catalog for data discovering [link to data catalog](https://eun-su1.azuredatacatalog.com/#/home)?
##### 	1.4.4	What is the data format e.g. structured in Database, free text or image?
##### 	1.4.5	Who owns the data?
##### 	1.4.6	How is the data accessed?
##### 	1.4.7	Who is the contact person to access the data?
##### 	1.4.8	What are the risks for accessing, understanding, analyzing the data?
##### 	1.4.9	What is the effort estimation for data collecting and preparing for each data source?
##### 	1.4.10	What are the data governance issues?
##### 	1.4.11	How is each use case/process/entity covered by the discovered data?

###	1.5	Identifying DS opportunity
##### 	1.5.1	What is the problem type from DS perspective, e.g. regression, classification, clustering etc.?
##### 	1.5.2	What is the problem-solving process from DS perspective (solution framework)?
##### 	1.5.3	What are the previous relevant experience/components that can be reused?
##### 	1.5.4	Has the feasibility study been done? What is the result?
##### 	1.5.5	What is the business value the DS solution can bring?
##### 	1.5.6	What is the consequence of a potential DS solution error? How can we control it?
##### 	1.5.7	How can the DS solution fit into the business process?
##### 	1.5.8	What are the main risks to fail? How can we control them?
##### 	1.5.9	What is the feedback from SME on the proposed DS solution?

###	1.6	Setting success/stop criteria
##### 	1.6.1	What is the evaluation strategy, objective evaluation or subjective evaluation?
##### 	1.6.2	What is the objective success criteria (recall, precision, accuracy, etc.)?
####	1.6.3	What is the subjective success criteria?
####	1.6.4	What is the stop criteria?
####	1.6.5	Have success/stop criteria communicated and agreed with stakeholders?
####	1.6.6	Is insufficient SME involvement part of the stop criteria? What has been agreed on with respect to the SME involvement and SME experience level?

###	1.7	Document review and artifact archiving
####	1.7.1	Has the document (answers to the questions) in this phase been reviewed and approved? If not, document the reason. (this question shall be answered the same number of times as the document review rounds for this phase)
####	1.7.2	What artifacts (document, code and data) are achieved in this phase? How are they archived for reuse and future reference?

##	2	Project Planning

###	2.1	Resource planning
####	2.1.1	What is the competence and resource plan?
####	2.1.2	What are the competences needed (competence matrix)?
####	2.1.3	How can team members meet the competence matrix?
####	2.1.4	Does team include the following roles: data scientist, ML engineer, and data engineer?
####	2.1.5	Are there dedicated SMEs allocated for the project?
####	2.1.6	What is the cooperation model with SME and end user?

###	2.2	Time planning
####	2.2.1	What is the time plan?
####	2.2.2	How is the DS time plan aligned with main project plan (if relevant)?
####	2.2.3	How does DS iterations fit into the time plan?
####	2.2.4	What are the definition of project phases/steps and what are milestones for them?
####	2.2.5	What are the risks to follow the plan? How can the risks be controlled?
####	2.2.6	What is the feedback from stakeholders on time plan?

###	2.3	Process planning
####	2.3.1	What is the scope of the DS project?
####	2.3.2	Is the DS project a standalone process or running parallel with a bigger SW implementation project where DS solution is part of?
####	2.3.3	If parallel, what are the cooperation model between the main process and the DS process?
####	2.3.4	If parallel, has DS project research/iteration feature communicated with the main project management?
####	2.3.5	Do you follow the rule: start with simple, get value into business and iteration for improvement?
####	2.3.6	How are the DS iterations planned in the project?
####	2.3.7	What is the plan to build up pipeline as early as possible to speed up the iteration?
####	2.3.8	Is Kanban agile project process considered as the first option? If not, why?
####	2.3.9	What is the feedback loop from stakeholders?
####	2.3.10	How can the results from each phase be reviewed by stakeholders?
####    2.3.11  Has an architecture contract been completed?

###	2.4	DS tools planning
####	2.4.1	What tools/platforms/systems are planned to be used in the DS project?
####	2.4.2	Are there experience/competence gap to use these tools? If yes, what is the plan to close the gap?
####	2.4.3	Are all the tools standard tools according to the DCOE DS tech standard? If not, why? ([link to the standard wiki page](https://wiki.statoil.no/wiki/index.php/Statoil_Data_Science_Technical_Standards))
####	2.4.4	What is the plan for code/document review, artifact archiving, and knowledge sharing?

###	2.5	Document review and artifact archiving
####	2.5.1	Has the document (answers to the questions) in this phase been reviewed and approved? If not, document the reason. (this question shall be answered the same number of times as the document review rounds for this phase)
####	2.5.2	What artifacts (document, code and data) are achieved in this phase? How are they archived for reuse and future reference?

##	3	Data collecting and preparing

###	3.1	Collecting data
####	3.1.1	What are the rules to select relevant and irrelevant data?
####	3.1.2	What is the frequency/granularity the data is collected? Is it enough for the target problem?
####	3.1.3	Is the raw data kept untouched after collection?
####	3.1.4	Is the data to be stored in the data platform? If not, why?
####	3.1.5	Is open data format used to store the data, for example, txt, json, csv? If not, why?
#### 3.1.6 Is an automated pipeline set up for processing new data? If not, why?
####	3.1.7	Document the data collection process?

###	3.2	Exploring data
####	3.2.1	What is the structure of the data?
####	3.2.2	What are the relationships between data items?
####	3.2.3	How are the data from different source mapped together?
####	3.2.4	What tools, statistical methods, visualization tools have been used to explore the data?
####	3.2.5	What is the data quality: completeness, consistency, validity, and accuracy?
####	3.2.6	What are the other issues with data quality?
####	3.2.7	How should duplicated data be filtered or removed?
####	3.2.8	Are there outliers in the data?
####	3.2.9	What patterns have you found in the data?

###	3.3	Understanding data from domain
####	3.3.1	How was the data generated?
####	3.3.2	How was the data sampled/transferred?
####	3.3.3	What is the meaning of each data item from business perspective?
####	3.3.4	What is the relationship between each data item and the target output?
####	3.3.5	Are there unstable data period, how to identify and remove?
####	3.3.6	Are all the identified outliers noise?
####	3.3.7	What is the valid data range for each data item?

###	3.4	Preparing data
####	3.4.1	What is the plan to prepare the data?
####	3.4.2	What is the feedback from SME on the data preparation plan?
####	3.4.3	How is missing data filled/removed?
####	3.4.4	How is noise data removed/replaced?
####	3.4.5	How is the overlapped data combined/filtered/removed?
####	3.4.6	How the data is transformed?
####	3.4.7	How are training, validation and test dataset split?

###	3.5	Feature engineering
####	3.5.1	Is ML method used to reduce the complexity of the input feature space, for example, PCA or autoencoder?
####	3.5.2	What are the useful domain characteristics that are not represented in the dataset?
####	3.5.3	What features can be identified/created to represent the identified missing characters?

###	3.6	Document review and artifact archiving
####	3.6.1	Has the document (answers to the questions) in this phase been reviewed and approved? If not, document the reason. (this question shall be answered the same number of times as the document review rounds for this phase)
####	3.6.2	What artifacts (document, code and data) are achieved in this phase? How are they archived for reuse and future reference?

##	4	Modeling

###	4.1	Selecting model
####	4.1.1	What DS models have been considered, and what are the ones chosen for further evaluation?
####	4.1.2	What are the advantages and disadvantages of the chosen models?
####	4.1.3	Is the simplest model chosen as benchmark in the first iteration?
####	4.1.4	What is the criteria to compare candidate models?

###	4.2	Building model
####	4.2.1	What are the hyper-parameters for the selected models?
####	4.2.2	What are the processes to optimize the hyper-parameters?
####	4.2.3	Has the data been normalized? Give explanation.
####	4.2.4	What is the time used to train the model? Is it acceptable for offline or online training?

###	4.3	Testing model
####	4.3.1	Is separate test dataset used to test models?
####	4.3.2	Are success criteria met by models?
####	4.3.3	How easily can the end user use/understand model output?
####	4.3.4	Is model performance part of the criteria?
####	4.3.5	What is the subjective evaluation result?
####	4.3.6	If more than one models meet the criteria, how should one/ones have chosen from them?

###	4.4	Document review and artifact archiving
####	4.4.1	Has the document (answers to the questions) in this phase been reviewed and approved? If not, document the reason. (this question shall be answered the same number of times as the document review rounds for this phase)
####	4.4.2	What artifacts (document, code and data) are achieved in this phase? How are they archived for reuse and future reference?

##	5	Evaluation

###	5.1	Technical evaluation
####	5.1.1	Have predefined success/acceptance criteria been met by the chosen model?
####	5.1.2	Have all the identified requirements been met?
####	5.1.3	Have all the identified use cases been covered?
####	5.1.4	What are the preconditions and limitations of the chosen model?
####	5.1.5	How easily can the model be integrated into the work process?
####	5.1.6	What are the end user competence needed to use the model? Is this competence requirement acceptable by the end user organization?
####	5.1.7	Has the business changed so that the solution cannot be applied anymore?

###	5.2	Process evaluation
####	5.2.1	Has the planned artifact peer review been executed as planed?
####	5.2.2	Have the communication channels with stakeholders worked as expected?
####	5.2.3	Has the time plan been met? If not, why?
####	5.2.4	Have all the relevant documents been in place and approved?
####	5.2.5	What is the go/no-go decision and reason behind it?
####	5.2.6	What is the feedback from SME or end user on the evaluation result?

###	5.3	Document review and artifact archiving
####	5.3.1	Has the document (answers to the questions) in this phase been reviewed and approved? If not, document the reason. (this question shall be answered the same number of times as the document review rounds for this phase)
####	5.3.2	What artifacts (document, code and data) are achieved in this phase? How are they archived for reuse and future reference?

##	6	Deployment and Monitoring

###	6.1	Deploying model
####	6.1.1	How is the chosen model implemented/deployed?
####	6.1.2	How is the result to be presented to the end user? Is there a graphic way to do it?
####	6.1.3	If deployed as SW, does the Statoil architecture contract been met?
####	6.1.4	Which TRL (Technology readiness level) level is the SW classified as?
####	6.1.5	How are the results to be interpreted or utilized?
####	6.1.6	Are there IP governance issues?

###	6.2	Managing the process change
####	6.2.1	What are the changes the new DS solution brings to the business process?
####	6.2.2	What are the efforts/process to adopt the new solution by the management?
####	6.2.3	What effort is used to train the end user to use the new solution?

###	6.3	Monitoring and maintaining DS model
####	6.3.1	What is the operation/maintenance plan for the DS solution?
####	6.3.2	How often shall the model be re-trained and re-deployed? Automatically or manually?
####	6.3.3	What is the usage monitoring plan for the DS solution?
####	6.3.4	Are the DS solution constraints/limitations under monitoring?
####	6.3.5	What is the user feedback loop?

###	6.4	Knowledge sharing
####	6.4.1	What can be learned from the project process?
####	6.4.2	How can the experience be shared with others and retrieved for future reference?
####	6.4.3	What DS components (knowledge, product, process and data) can we reuse or share?
####	6.4.4	What improvement suggestions do you have for this DS standard template?

###	6.5	Document review
####	6.5.1	Has the document (answers to the questions) in this phase been reviewed and approved? If not, document the reason. (this question shall be answered the same number of times as the document review rounds for this phase)


text:
{{cookiecutter.project_name}}
======================================================================

.. note::
   This documentation page is for your own use as you best see fit for your project.

   In some cases it might be enough with the README.md in the project root, however
   you might use this if you want to publish API documentation, or have a website
   where you want to make detailed project information available (you can e.g.
   publish direct from blob storage).

   To generate documentation install make and from the docs folder run:

      .. code-block::

         make html

   On Windows you can use the .bat file so from the docs folder just run:

      .. code-block::

         make html

Usage and setup
---------------

Information about this project including steps on how to setup, run examples
to reproduce results, and other guidelines.

.. note::
   Here you might include information about this project including steps on how to
   setup and reproduce results and findings, and other guidelines. As default we
   include the Equinor code of conduct, process documentation and any .rst files
   under the info folder. Edit / add / remove as needed. The table of contents is
   generated automatically based upon the referenced document headings.

.. toctree::
   :glob:
   :maxdepth: 2

   info/*

Results and findings
--------------------

Results and findings generated during the course of this project.

.. note::
   Here you might include a write up of results or links to notebooks or other
   information that contain results or other findings. As default we
   include any .rst files under the results folder. Edit / add / remove as needed.
   The table of contents is generated automatically based upon the referenced
   document headings.

.. toctree::
   :glob:
   :maxdepth: 2

   results/*

API Documentation
-----------------

Information on the underlying API including function, class and method
documentation.

.. note::
   If you don't want this, then your project probably isn't written according
   to best practices and likely not production ready. If you disagree, just
   edit and remove this section.

.. toctree::
   :maxdepth: 2

   api-{{cookiecutter.package_name}}

Indices and tables
------------------

* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`


text:
@ECHO OFF

pushd %~dp0

REM Command file for Sphinx documentation

if "%SPHINXBUILD%" == "" (
	set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=.
set BUILDDIR=_build

if "%1" == "" goto help

%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
	echo.
	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
	echo.installed, then set the SPHINXBUILD environment variable to point
	echo.to the full path of the 'sphinx-build' executable. Alternatively you
	echo.may add the Sphinx directory to PATH.
	echo.
	echo.If you don't have Sphinx installed, grab it from
	echo.http://sphinx-doc.org/
	exit /b 1
)

%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
goto end

:help
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%

:end
popd


text:
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# http://www.sphinx-doc.org/en/master/config

# -- Path setup --------------------------------------------------------------

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
sys.path.insert(0, os.path.abspath(os.path.join('..', '..', 'src')))

# -- Project information -----------------------------------------------------

project = '{{cookiecutter.project_name}}'
copyright = 'Equinor'
author = '{{cookiecutter.project_name}}'

# The full version, including alpha/beta/rc tags
release = 'version'

# -- General configuration ---------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autosummary',
    'sphinx.ext.napoleon',
    'm2r'
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']

# Extensions that recomonmark will use
source_suffix = {
    '.rst': 'restructuredtext',
    '.txt': 'markdown',
    '.md': 'markdown',
}

# Auto generate API documentation for modules
autosummary_generate = True

# Default flags used by autodoc directives
autodoc_default_options = {
    'members': True,
    'member-order': 'bysource',
    'special-members': '__init__',
}

# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
# html_theme = 'sphinx_rtd_theme'
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
# alabaster theme options: https://alabaster.readthedocs.io/en/latest/customization.html
html_theme_options = {
    "description": "{{cookiecutter.project_description}}",
    "extra_nav_links": {
        "Index": "genindex.html",
        "Module Index": "py-modindex.html",
        "Search Page": "search.html"
    },
    "github_banner": False,
    "note_bg": "#FFF59C",
    "show_powered_by": False,
    "show_related": False,
    "sidebar_collapse": False,
}

# Custom sidebar templates (often theme specific), maps document names to template names.
# alabaster options: https://alabaster.readthedocs.io/en/latest/customization.html
html_sidebars = {
    "index": [
        "about.html",
        "navigation.html",
        "searchbox.html"
    ],
    "**": [
        "about.html",
        'navigation.html',
        "searchbox.html"
    ],
}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = [
    # "_static"
]

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
# html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
# html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
html_use_smartypants = False

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
# html_domain_indices = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
html_show_sourcelink = False

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
html_show_sphinx = False

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
html_show_copyright = True


text:
api-{{cookiecutter.package_name}} package
================================================================================

This page contains information about the mhewtest package.

.. note::
   Edit this page to add additional description above and list whatever modules
   you would like included below. Documentation pages for the individual modules
   will be automatically generated when you 'make' the documentation.

.. autosummary::
   :toctree: _generated

   {{cookiecutter.package_name}}.examplemodule
   {{cookiecutter.package_name}}.features


text:
# Minimal makefile for Sphinx documentation
#

# You can set these variables from the command line, and also
# from the environment for the first two.
SPHINXOPTS    ?=
SPHINXBUILD   ?= sphinx-build
SOURCEDIR     = .
BUILDDIR      = _build

# Put it first so that "make" without argument is like "make help".
help:
	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

.PHONY: help Makefile

# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)


text:
Exploratory Analysis
====================

Introduction
------------

.. note::
   These documentation page are for your own use as you best see fit for your project.

   Here you could add information and links to present results from EDA or
   one of your other experiments. This might include links back to notebooks
   or other artifacts.


text:
Setup
=====

Introduction
------------

.. note::
   These documentation page are for your own use as you best see fit for your project.

   Here you could add setup information, or details on how you run things from an
   operational perspective.


text:
"""
This module should contain project specific io functionality.

Loading and saving of files should be deferred to this class for easy and consistent file handling between different
sources and to have a single location where file references are held.
"""
import os
import pandas as pd
from statoilds import datalake

class IO:
    local_data_path = '.'

    def __init__(self, local_data_path: str):
        """
        Constructor that can set the data path from where we will access local data..

        Args:
            path: Path to the data folder.
        """
        self.local_data_path = local_data_path

    def load_cleaned_file(self, download_always: bool = True):
        """
        Load the cleaned file, optionally logging into to Azure to download.

        If token is passed then this will only login if token isn't already valid

        Args:
            download_always: Whether to always download the file even if it exists locally

        Returns:
            A dataframe used for logging in and the login token
        """
        local_path = os.path.join(self.local_data_path, self.cleaned_file_local)

        token = datalake.login_and_download_file(self.cleaned_file_remote,
                                                 local_path,
                                                 download_always=download_always)

        df = pd.read_csv(local_path,
                         dtype={'Well_name': 'category'},
                         parse_dates=['start'])
        return df


text:
"""
This module should contain your main project pipeline(s).

Whilst the pipeline may change during the analysis phases, any more stable pipeline should be implemented here so
that it can be reused and easily reproduced.
"""
# import pandas as pd

# from examplepackage import features
# from examplepackage.io import IO

def run_pipeline(local_data_path: str):
    """
    Run the main processing pipeline.

    Returns:
        A dataframe containing the output of the pipeline
    """

    # io = IO(path)
    # df = io.load_cleaned_file(download_always=False)
    # df = add_choke_events(df)

    # Add calls to features.Xxx here

    # save (or return) dataframe here?


text:
"""
This module should contain project specific feature engineering functionality.

You should avoid engineering features in a notebook as it is not transferable later if you want to automate the
process. Add functions here to create your features, such functions should include those to generate specific features
along with any more generic functions.

Consider moving generic functions into the shared statoilds package.
"""
import pandas as pd

def my_feature_xxx(df: pd.DataFrame):
    """
    Description goes here.
    You might also add additional arguments such as column etc...
    Would be nice with some test cases also :)

    Args:
        df: Dataframe upon which to operate

    Returns:
        A dataframe with the Xxx feature appended
    """

    # CODE HERE

    return df


text:
"""
This example module shows some simple methods and best practices for documentation
"""

import numpy as np

def hello_world() -> str:
    """
    Method description - A simple method to get the hello world string

    Returns:
       The string "Hello World"
    """
    return "Hello World"

def add_value_to_numpy(array: np.ndarray, amount: float = 1) -> np.ndarray:
    """
    A sample method to add a value to every element in a pandas DataFrame.

    Args:
        array: The source DataFrame to work on.
        amount: The amount to add to each element in the DataFrame

    Returns:
        A new DataFrame with each value increased by amount.

    Examples:
        Examples should be written in doctest format, and should illustrate how
        to use the function.
        >>> array = np.array([1, 1, 1 ,1, 1])
        >>> result_array = add_value_to_numpy(array, 1)

    """
    if array is None or \
            not isinstance(array, np.ndarray):
        raise ValueError("array must be a valid ndarray")
    # if isinstance(a, np.ndarray):

    return array + amount


text:
#!/usr/bin/env python

"""An example python script template.

Argv:
    infile: Input file
    outfile: Output file

"""

import sys
import argparse

def main(arguments):

    parser = argparse.ArgumentParser(
        description="Put your description here",
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('infile', help="Input file", type=argparse.FileType('r'))
    parser.add_argument('-o', '--outfile', help="Output file",
                        default=sys.stdout, type=argparse.FileType('w'))

    args = parser.parse_args(arguments)

    print(args)

    # Add your code here

if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))


text:
#!/usr/bin/env python

"""Model scoring (WIP) - Contributions welcome!!
"""
# import argparse
import joblib
import json
import numpy

from azureml.core.model import Model

from sklearn import datasets
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split

def init():
    global model
    model_path = Model.get_model_path('{{cookiecutter.mlops_name}}')
    # deserialize the model file back into a sklearn model
    model = joblib.load(model_path)

# note you can pass in multiple rows for scoring
def run(raw_data):
    try:
        data = json.loads(raw_data)['data']
        data = numpy.array(data)
        result = model.predict(data)
        # you can return any datatype if it is JSON-serializable
        return result.tolist()
    except Exception as e:
        error = str(e)
        return error

def main():
    # parser = argparse.ArgumentParser()
    # environment parameters
    # parser.add_argument(
    #     '--data-folder',
    #     help="local path to training data",
    #     required=True
    # )
    # parser.add_argument(
    #     "--output-dir", type=str, default=os.path.join('..', 'outputs'),
    #     help='location to writeoutput relative to this script'
    # )

    # parse the arguments
    # args = parser.parse_args()

    # ws = Workspace.from_config()
    # model = Model(ws, 'sklearn_mnist')

    # model.download(target_dir=os.getcwd(), exist_ok=True)

    # verify the downloaded model file
    file_path = "ml-service/{{cookiecutter.mlops_name}}.joblib"
    model = joblib.load(file_path)

    # loading the iris dataset
    iris = datasets.load_iris()

    # X -> features, y -> label
    X = iris.data
    y = iris.target

    # dividing X, y into train and test data
    _, X_test, _, y_test = train_test_split(X, y, random_state=0)

    # training a linear SVM classifier
    y_pred = model.predict(X_test)

    # model accuracy
    accuracy = accuracy_score(y_test, y_pred)
    print('Accuracy of SVM classifier on test set: {:.2f}'.format(accuracy))

if __name__ == '__main__':
    main()


text:
#!/usr/bin/env python

"""Train a model locally using Azure ML.

This will re-use the current python environment.

Argv:
    output-dir: A folder to store any output to
    kernel: Kernel type to be used in the algorithm
    penalty: Penalty parameter of the error term
"""
import argparse
import sys

import azureml.core
from azureml.core import Experiment, ScriptRunConfig, Workspace

def submit(experiment_name: str,
           kernal: str,
           penalty: float):

    print("This notebook was created using version 1.0.83 of the Azure ML SDK")
    print("You are using version", azureml.core.VERSION, "of the SDK")

    # Get a reference to the workspace. Be sure to download the config.json
    # from your workspace and place in the parent folder.
    ws = Workspace.from_config()
    print('Loaded workspace', ws.name)

    # Reference the experiment
    experiment = Experiment(workspace=ws, name=experiment_name)
    print('Logging to experiment', experiment_name)

    # Create the RunConfiguration that will be used
    arguments = [
        '--output-dir', "outputs",
        '--kernel', kernal,
        '--penalty', penalty,
    ]
    script_run_config = ScriptRunConfig(source_directory='.',
                                        script='train.py',
                                        arguments=arguments)

    # As we will run locally we can use our existing python environment
    script_run_config.run_config.environment. \
        python.user_managed_dependencies = True

    # Submit the experiment to get a run and wait for completion
    run = experiment.submit(script_run_config)
    print('Submitted please wait...')
    run.wait_for_completion(show_output=True)

    # register the trained model
    model = run.register_model(
        model_name='{{cookiecutter.mlops_name}}',
        model_path='outputs/model/{{cookiecutter.mlops_name}}.joblib')

    print('Run number:', run.number)
    print('Run id:', run.id)
    print("Run details are available at:", run.get_portal_url())
    print("Model: {} v{}".format(model.name, model.version))

    if 'azureml.git.dirty' in run.properties:
        if run.properties['azureml.git.dirty']:
            print("WARNNG: You have uncomitted changes. To ensure "
                  "reproducability check in your code before you train.")
    else:
        print('WARNNG: To ensure reproducability you should be using git!')

def main(arguments: list):
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    # environment parameters
    parser.add_argument('--experiment', type=str,
                        default='{{cookiecutter.mlops_name}}-local',
                        help='The name of the Azure ML Experiment')

    # training specific parameters
    parser.add_argument('--kernel', type=str, default='linear',
                        help='Kernel type to be used in the algorithm')
    parser.add_argument('--penalty', type=float, default=1.0,
                        help='Penalty parameter of the error term')

    # parse the arguments
    args = parser.parse_args(arguments)

    # submit the job
    submit(args.experiment,
           args.kernel,
           args.penalty)

if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))


text:
#!/usr/bin/env python

"""Train a model remotely using Azure ML compute.

This will re-use the current python environment.

Argv:
    output-dir: A folder to store any output to
    kernel: Kernel type to be used in the algorithm
    penalty: Penalty parameter of the error term
"""
import argparse
import sys

import azureml.core
from azureml.core import Experiment, Workspace
from azureml.train.sklearn import SKLearn

def submit(experiment_name: str,
           compute_name: str,
           kernal: str,
           penalty: float):

    print("This notebook was created using version 1.0.83 of the Azure ML SDK")
    print("You are using version", azureml.core.VERSION, "of the SDK")

    # Get a reference to the workspace. Be sure to download the config.json
    # from your workspace and place in the parent folder.
    ws = Workspace.from_config()
    print('Loaded workspace', ws.name)

    # Reference the experiment
    experiment = Experiment(workspace=ws, name=experiment_name)
    print('Logging to experiment', experiment_name)

    # Get a reference to an existing the compute target.
    compute_target = ws.compute_targets[compute_name]

    # Setup an Estimator for submitting the job. An Estimator further wraps
    # RunConfig with additional configuration for specific cases. There are
    # Estimators provided for many common runtimes such as PyTorch and
    # Tensorflow. In this case we use the SKLearn specific estimator.
    script_params = {
        '--output-dir': "outputs",
        '--kernel': kernal,
        '--penalty': penalty,
    }

    # NOTE: scikit-learn added below until default image includes v22.1+
    estimator = SKLearn(source_directory=".",
                        entry_script='train.py',
                        script_params=script_params,
                        compute_target=compute_target,
                        pip_packages=['matplotlib', 'scikit-learn'])

    # Submit the experiment to get a run and wait for completion
    run = experiment.submit(estimator)
    print('Submitted please wait...')
    run.wait_for_completion(show_output=True)

    # register the trained model
    model = run.register_model(
        model_name='{{cookiecutter.mlops_name}}',
        model_path='outputs/model/{{cookiecutter.mlops_name}}.joblib')

    print('Run number:', run.number)
    print('Run id:', run.id)
    print("Run details are available at:", run.get_portal_url())
    print("Model: {} v{}".format(model.name, model.version))

    if 'azureml.git.dirty' in run.properties:
        if run.properties['azureml.git.dirty']:
            print("WARNNG: You have uncomitted changes. To ensure "
                  "reproducability check in your code before you train.")
    else:
        print('WARNNG: To ensure reproducability you should be using git!')

def main(arguments: list):
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    # environment parameters
    parser.add_argument('--experiment', type=str,
                        default='{{cookiecutter.mlops_name}}',
                        help='The name of the Azure ML Experiment')
    parser.add_argument('--compute-name', type=str,
                        default='{{cookiecutter.mlops_compute_name}}',
                        help='The name of the Azure ML compute cluster')

    # training specific parameters
    parser.add_argument('--kernel', type=str, default='linear',
                        help='Kernel type to be used in the algorithm')
    parser.add_argument('--penalty', type=float, default=1.0,
                        help='Penalty parameter of the error term')

    # parse the arguments
    args = parser.parse_args(arguments)

    # submit the job
    submit(args.experiment,
           args.compute_name,
           args.kernel,
           args.penalty)

if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))


text:
try:
    from azureml.core import Run
except ImportError:
    pass

# access the Azure ML run
# init run param to check if running within AML
def get_AMLRun():
    """Try and get the Azure Machine Learning run

    Returns:
        Run: The Experiment run or None if no active run.
    """
    try:
        run = Run.get_context(allow_offline=False)
        return run
    except Exception as e:
        print("Caught = {}".format(e.message))
        return None