digital-forensics-labs/SecGen
1
0
Fork 0
mirror of https://github.com/cliffe/SecGen.git synced 2026-02-23 04:08:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

example code: python2 and python3 install modules, script generators and ctf vulnerability/challenge module wrappers

Browse Source
This commit is contained in:
ts
2018-08-03 15:32:13 +01:00
parent b8b27d937c
commit 1dd5bf6f79
79 changed files with 3796 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
modules/utilities/unix/languages/python2/python/manifests/config.pp Normal file
View File

@@ -0,0 +1,40 @@
# == Define: python::config
#
# Optionally installs the gunicorn service
#
# === Examples
#
# include python::config
#
# === Authors
#
# Sergey Stankevich
# Ashley Penney
# Fotis Gimian
#
class python::config {
Class['python::install'] -> Python::Pip <| |>
Class['python::install'] -> Python::Requirements <| |>
Class['python::install'] -> Python::Virtualenv <| |>
Python::Virtualenv <| |> -> Python::Pip <| |>
if $python::manage_gunicorn {
if $python::gunicorn != 'absent' {
Class['python::install'] -> Python::Gunicorn <| |>
Python::Gunicorn <| |> ~> Service['gunicorn']
service { 'gunicorn':
ensure => running,
enable => true,
hasrestart => true,
hasstatus => false,
pattern => '/usr/bin/gunicorn',
}
}
}
}

62
modules/utilities/unix/languages/python2/python/manifests/dotfile.pp Normal file
View File

@@ -0,0 +1,62 @@
# == Define: python::dotfile
#
# Manages any python dotfiles with a simple config hash.
#
# === Parameters
#
# [*ensure*]
# present|absent. Default: present
#
# [*filename*]
# Filename. Default: $title
#
# [*mode*]
# File mode. Default: 0644
#
# [*owner*]
# [*group*]
# Owner/group. Default: `root`/`root`
#
# [*config*]
# Config hash. This will be expanded to an ini-file. Default: {}
#
# === Examples
#
# python::dotfile { '/var/lib/jenkins/.pip/pip.conf':
# ensure => present,
# owner => 'jenkins',
# group => 'jenkins',
# config => {
# 'global' => {
# 'index-url => 'https://mypypi.acme.com/simple/'
# 'extra-index-url => https://pypi.risedev.at/simple/
# }
# }
# }
#
#
define python::dotfile (
$ensure = 'present',
$filename = $title,
$owner = 'root',
$group = 'root',
$mode = '0644',
$config = {},
) {
$parent_dir = dirname($filename)
exec { "create ${title}'s parent dir":
command => "install -o ${owner} -g ${group} -d ${parent_dir}",
path => [ '/usr/bin', '/bin', '/usr/local/bin', ],
creates => $parent_dir,
}
file { $filename:
ensure => $ensure,
owner => $owner,
group => $group,
mode => $mode,
content => template("${module_name}/inifile.erb"),
require => Exec["create ${title}'s parent dir"],
}
}

132
modules/utilities/unix/languages/python2/python/manifests/gunicorn.pp Normal file
View File

@@ -0,0 +1,132 @@
# == Define: python::gunicorn
#
# Manages Gunicorn virtual hosts.
#
# === Parameters
#
# [*ensure*]
# present|absent. Default: present
#
# [*config_dir*]
# Configure the gunicorn config directory path. Default: /etc/gunicorn.d
#
# [*manage_config_dir*]
# Set if the gunicorn config directory should be created. Default: false
#
# [*virtualenv*]
# Run in virtualenv, specify directory. Default: disabled
#
# [*mode*]
# Gunicorn mode.
# wsgi|django. Default: wsgi
#
# [*dir*]
# Application directory.
#
# [*bind*]
# Bind on: 'HOST', 'HOST:PORT', 'unix:PATH'.
# Default: system-wide: unix:/tmp/gunicorn-$name.socket
# virtualenv: unix:${virtualenv}/${name}.socket
#
# [*environment*]
# Set ENVIRONMENT variable. Default: none
#
# [*appmodule*]
# Set the application module name for gunicorn to load when not using Django.
# Default: app:app
#
# [*osenv*]
# Allows setting environment variables for the gunicorn service. Accepts a
# hash of 'key': 'value' pairs.
# Default: false
#
# [*timeout*]
# Allows setting the gunicorn idle worker process time before being killed.
# The unit of time is seconds.
# Default: 30
#
# [*template*]
# Which ERB template to use. Default: python/gunicorn.erb
#
# [*args*]
# Custom arguments to add in gunicorn config file. Default: []
#
# === Examples
#
# python::gunicorn { 'vhost':
# ensure => present,
# virtualenv => '/var/www/project1',
# mode => 'wsgi',
# dir => '/var/www/project1/current',
# bind => 'unix:/tmp/gunicorn.socket',
# environment => 'prod',
# owner => 'www-data',
# group => 'www-data',
# appmodule => 'app:app',
# osenv => { 'DBHOST' => 'dbserver.example.com' },
# timeout => 30,
# template => 'python/gunicorn.erb',
# }
#
# === Authors
#
# Sergey Stankevich
# Ashley Penney
# Marc Fournier
#
define python::gunicorn (
$ensure = present,
$config_dir = '/etc/gunicorn.d',
$manage_config_dir = false,
$virtualenv = false,
$mode = 'wsgi',
$dir = false,
$bind = false,
$environment = false,
$owner = 'www-data',
$group = 'www-data',
$appmodule = 'app:app',
$osenv = false,
$timeout = 30,
$workers = false,
$access_log_format = false,
$accesslog = false,
$errorlog = false,
$log_level = 'error',
$template = 'python/gunicorn.erb',
$args = [],
) {
# Parameter validation
if ! $dir {
fail('python::gunicorn: dir parameter must not be empty')
}
validate_re($log_level, 'debug|info|warning|error|critical', "Invalid \$log_level value ${log_level}")
if $manage_config_dir {
file { $config_dir:
ensure => directory,
mode => '0755',
owner => 'root',
group => 'root',
}
file { "${config_dir}/${name}":
ensure => $ensure,
mode => '0644',
owner => 'root',
group => 'root',
content => template($template),
require => File[$config_dir],
}
} else {
file { "${config_dir}/${name}":
ensure => $ensure,
mode => '0644',
owner => 'root',
group => 'root',
content => template($template),
}
}
}

150
modules/utilities/unix/languages/python2/python/manifests/init.pp Normal file
View File

@@ -0,0 +1,150 @@
# == Class: python
#
# Installs and manages python, python-dev, python-virtualenv and Gunicorn.
#
# === Parameters
#
# [*ensure*]
# Desired installation state for the Python package. Valid options are absent,
# present and latest. Default: present
#
# [*version*]
# Python version to install. Beware that valid values for this differ a) by
# the provider you choose and b) by the osfamily/operatingsystem you are using.
# Default: system default
# Allowed values:
# - provider == pip: everything pip allows as a version after the 'python=='
# - else: 'system', 'pypy', 3/3.3/...
# - Be aware that 'system' usually means python 2.X.
# - 'pypy' actually lets us use pypy as python.
# - 3/3.3/... means you are going to install the python3/python3.3/...
# package, if available on your osfamily.
#
# [*pip*]
# Desired installation state for python-pip. Boolean values are deprecated.
# Default: present
# Allowed values: 'absent', 'present', 'latest'
#
# [*dev*]
# Desired installation state for python-dev. Boolean values are deprecated.
# Default: absent
# Allowed values: 'absent', 'present', 'latest'
#
# [*virtualenv*]
# Desired installation state for python-virtualenv. Boolean values are
# deprecated. Default: absent
# Allowed values: 'absent', 'present', 'latest
#
# [*gunicorn*]
# Desired installation state for Gunicorn. Boolean values are deprecated.
# Default: absent
# Allowed values: 'absent', 'present', 'latest'
#
# [*manage_gunicorn*]
# Allow Installation / Removal of Gunicorn. Default: true
#
# [*provider*]
# What provider to use for installation of the packages, except gunicorn and
# Python itself. Default: system default provider
# Allowed values: 'pip'
#
# [*use_epel*]
# Boolean to determine if the epel class is used. Default: true
#
# === Examples
#
# class { 'python':
# version => 'system',
# pip => 'present',
# dev => 'present',
# virtualenv => 'present',
# gunicorn => 'present',
# }
#
# === Authors
#
# Sergey Stankevich
# Garrett Honeycutt <code@garretthoneycutt.com>
#
class python (
$ensure = $python::params::ensure,
$version = $python::params::version,
$pip = $python::params::pip,
$dev = $python::params::dev,
$virtualenv = $python::params::virtualenv,
$gunicorn = $python::params::gunicorn,
$manage_gunicorn = $python::params::manage_gunicorn,
$gunicorn_package_name = $python::params::gunicorn_package_name,
$provider = $python::params::provider,
$valid_versions = $python::params::valid_versions,
$python_pips = { },
$python_virtualenvs = { },
$python_pyvenvs = { },
$python_requirements = { },
$python_dotfiles = { },
$use_epel = $python::params::use_epel,
$rhscl_use_public_repository = $python::params::rhscl_use_public_repository,
) inherits python::params{
if $provider != undef and $provider != '' {
validate_re($provider, ['^(pip|scl|rhscl)$'],
"Only 'pip', 'rhscl' and 'scl' are valid providers besides the system default. Detected provider is <${provider}>.")
}
$exec_prefix = $provider ? {
'scl' => "/usr/bin/scl enable ${version} -- ",
'rhscl' => "/usr/bin/scl enable ${version} -- ",
default => '',
}
validate_re($ensure, ['^(absent|present|latest)$'])
validate_re($version, concat(['system', 'pypy'], $valid_versions))
if $pip == false or $pip == true {
warning('Use of boolean values for the $pip parameter is deprecated')
} else {
validate_re($pip, ['^(absent|present|latest)$'])
}
if $virtualenv == false or $virtualenv == true {
warning('Use of boolean values for the $virtualenv parameter is deprecated')
} else {
validate_re($virtualenv, ['^(absent|present|latest)$'])
}
if $virtualenv == false or $virtualenv == true {
warning('Use of boolean values for the $virtualenv parameter is deprecated')
} else {
validate_re($virtualenv, ['^(absent|present|latest)$'])
}
if $gunicorn == false or $gunicorn == true {
warning('Use of boolean values for the $gunicorn parameter is deprecated')
} else {
validate_re($gunicorn, ['^(absent|present|latest)$'])
}
validate_hash($python_dotfiles)
validate_bool($manage_gunicorn)
validate_bool($use_epel)
# Module compatibility check
$compatible = [ 'Debian', 'RedHat', 'Suse', 'Gentoo' ]
if ! ($::osfamily in $compatible) {
fail("Module is not compatible with ${::operatingsystem}")
}
# Anchor pattern to contain dependencies
anchor { 'python::begin': }
-> class { 'python::install': }
-> class { 'python::config': }
-> anchor { 'python::end': }
# Allow hiera configuration of python resources
create_resources('python::pip', $python_pips)
create_resources('python::pyvenv', $python_pyvenvs)
create_resources('python::virtualenv', $python_virtualenvs)
create_resources('python::requirements', $python_requirements)
create_resources('python::dotfile', $python_dotfiles)
}

255
modules/utilities/unix/languages/python2/python/manifests/install.pp Normal file
View File

@@ -0,0 +1,255 @@
# == Class: python::install
#
# Installs core python packages,
#
# === Examples
#
# include python::install
#
# === Authors
#
# Sergey Stankevich
# Ashley Penney
# Fotis Gimian
# Garrett Honeycutt <code@garretthoneycutt.com>
#
class python::install {
$python = $::python::version ? {
'system' => 'python',
'pypy' => 'pypy',
default => "${python::version}", # lint:ignore:only_variable_string
}
$pythondev = $::osfamily ? {
'RedHat' => "${python}-devel",
'Debian' => "${python}-dev",
'Suse' => "${python}-devel",
'Gentoo' => undef,
}
$dev_ensure = $python::dev ? {
true => 'present',
false => 'absent',
default => $python::dev,
}
$pip_ensure = $python::pip ? {
true => 'present',
false => 'absent',
default => $python::pip,
}
$venv_ensure = $python::virtualenv ? {
true => 'present',
false => 'absent',
default => $python::virtualenv,
}
package { 'python':
ensure => $python::ensure,
name => $python,
}
package { 'virtualenv':
ensure => $venv_ensure,
require => Package['python'],
}
case $python::provider {
pip: {
package { 'pip':
ensure => $pip_ensure,
require => Package['python'],
}
if $pythondev {
package { 'python-dev':
ensure => $dev_ensure,
name => $pythondev,
}
}
# Install pip without pip, see https://pip.pypa.io/en/stable/installing/.
exec { 'bootstrap pip':
command => '/usr/bin/curl https://bootstrap.pypa.io/get-pip.py | python',
unless => 'which pip',
path => [ '/bin', '/usr/bin', '/usr/local/bin' ],
require => Package['python'],
}
# Puppet is opinionated about the pip command name
file { 'pip-python':
ensure => link,
path => '/usr/bin/pip-python',
target => '/usr/bin/pip',
require => Exec['bootstrap pip'],
}
Exec['bootstrap pip'] -> File['pip-python'] -> Package <| provider == pip |>
Package <| title == 'pip' |> {
name => 'pip',
provider => 'pip',
}
Package <| title == 'virtualenv' |> {
name => 'virtualenv',
provider => 'pip',
}
}
scl: {
# SCL is only valid in the RedHat family. If RHEL, package must be
# enabled using the subscription manager outside of puppet. If CentOS,
# the centos-release-SCL will install the repository.
$install_scl_repo_package = $::operatingsystem ? {
'CentOS' => 'present',
default => 'absent',
}
package { 'centos-release-scl':
ensure => $install_scl_repo_package,
before => Package['scl-utils'],
}
package { 'scl-utils':
ensure => 'latest',
before => Package['python'],
}
# This gets installed as a dependency anyway
# package { "${python::version}-python-virtualenv":
# ensure => $venv_ensure,
# require => Package['scl-utils'],
# }
package { "${python}-scldevel":
ensure => $dev_ensure,
require => Package['scl-utils'],
}
if $pip_ensure != 'absent' {
exec { 'python-scl-pip-install':
command => "${python::exec_prefix}easy_install pip",
path => ['/usr/bin', '/bin'],
creates => "/opt/rh/${python::version}/root/usr/bin/pip",
require => Package['scl-utils'],
}
}
}
rhscl: {
# rhscl is RedHat SCLs from softwarecollections.org
if $::python::rhscl_use_public_repository {
$scl_package = "rhscl-${::python::version}-epel-${::operatingsystemmajrelease}-${::architecture}"
package { $scl_package:
source => "https://www.softwarecollections.org/en/scls/rhscl/${::python::version}/epel-${::operatingsystemmajrelease}-${::architecture}/download/${scl_package}.noarch.rpm",
provider => 'rpm',
tag => 'python-scl-repo',
}
}
Package <| title == 'python' |> {
tag => 'python-scl-package',
}
Package <| title == 'virtualenv' |> {
name => "${python}-python-virtualenv",
}
package { "${python}-scldevel":
ensure => $dev_ensure,
tag => 'python-scl-package',
}
package { "${python}-python-pip":
ensure => $pip_ensure,
tag => 'python-pip-package',
}
if $::python::rhscl_use_public_repository {
Package <| tag == 'python-scl-repo' |>
-> Package <| tag == 'python-scl-package' |>
}
Package <| tag == 'python-scl-package' |>
-> Package <| tag == 'python-pip-package' |>
}
default: {
package { 'pip':
ensure => $pip_ensure,
require => Package['python'],
}
if $pythondev {
package { 'python-dev':
ensure => $dev_ensure,
name => $pythondev,
alias => $pythondev,
}
}
if $::osfamily == 'RedHat' {
if $pip_ensure != 'absent' {
if $python::use_epel == true {
include 'epel'
Class['epel'] -> Package['pip']
}
}
if ($venv_ensure != 'absent') and ($::operatingsystemrelease =~ /^6/) {
if $python::use_epel == true {
include 'epel'
Class['epel'] -> Package['virtualenv']
}
}
$virtualenv_package = "${python}-virtualenv"
} else {
if $::lsbdistcodename == 'jessie' {
$virtualenv_package = 'virtualenv'
} elsif $::lsbdistcodename == 'stretch' {
$virtualenv_package = 'virtualenv'
} elsif $::lsbdistcodename == 'xenial' {
$virtualenv_package = 'virtualenv'
} elsif $::osfamily == 'Gentoo' {
$virtualenv_package = 'virtualenv'
} else {
$virtualenv_package = 'python-virtualenv'
}
}
if "${::python::version}" =~ /^3/ { #lint:ignore:only_variable_string
$pip_category = undef
$pip_package = 'python3-pip'
} elsif ($::osfamily == 'RedHat') and (versioncmp($::operatingsystemmajrelease, '7') >= 0) {
$pip_category = undef
$pip_package = 'python2-pip'
} elsif $::osfamily == 'Gentoo' {
$pip_category = 'dev-python'
$pip_package = 'pip'
} else {
$pip_category = undef
$pip_package = 'python-pip'
}
Package <| title == 'pip' |> {
name => $pip_package,
category => $pip_category,
}
Package <| title == 'virtualenv' |> {
name => $virtualenv_package,
}
}
}
if $python::manage_gunicorn {
$gunicorn_ensure = $python::gunicorn ? {
true => 'present',
false => 'absent',
default => $python::gunicorn,
}
package { 'gunicorn':
ensure => $gunicorn_ensure,
name => $python::gunicorn_package_name,
}
}
}

38
modules/utilities/unix/languages/python2/python/manifests/params.pp Normal file
View File

@@ -0,0 +1,38 @@
# == Class: python::params
#
# The python Module default configuration settings.
#
class python::params {
$ensure = 'present'
$version = 'system'
$pip = 'present'
$dev = 'absent'
$virtualenv = 'absent'
$gunicorn = 'absent'
$manage_gunicorn = true
$provider = undef
$valid_versions = $::osfamily ? {
'RedHat' => ['3','27','33'],
'Debian' => ['3', '3.3', '2.7'],
'Suse' => [],
'Gentoo' => ['2.7', '3.3', '3.4', '3.5']
}
if $::osfamily == 'RedHat' {
if $::operatingsystem != 'Fedora' {
$use_epel = true
} else {
$use_epel = false
}
} else {
$use_epel = false
}
$gunicorn_package_name = $::osfamily ? {
'RedHat' => 'python-gunicorn',
default => 'gunicorn',
}
$rhscl_use_public_repository = true
}

275
modules/utilities/unix/languages/python2/python/manifests/pip.pp Normal file
View File

@@ -0,0 +1,275 @@
# == Define: python::pip
#
# Installs and manages packages from pip.
#
# === Parameters
#
# [*name]
# must be unique
#
# [*pkgname]
# name of the package. If pkgname is not specified, use name (title) instead.
#
# [*ensure*]
# present|absent. Default: present
#
# [*virtualenv*]
# virtualenv to run pip in.
#
# [*url*]
# URL to install from. Default: none
#
# [*owner*]
# The owner of the virtualenv being manipulated. Default: root
#
# [*group*]
# The group of the virtualenv being manipulated. Default: root
#
# [*index*]
# Base URL of Python package index. Default: none (http://pypi.python.org/simple/)
#
# [*proxy*]
# Proxy server to use for outbound connections. Default: none
#
# [*editable*]
# Boolean. If true the package is installed as an editable resource.
#
# [*environment*]
# Additional environment variables required to install the packages. Default: none
#
# [*timeout*]
# The maximum time in seconds the "pip install" command should take. Default: 1800
#
# [*install_args*]
# String. Any additional installation arguments that will be supplied
# when running pip install.
#
# [*uninstall_args*]
# String. Any additional arguments that will be supplied when running
# pip uninstall.
#
# [*log_dir*]
# String. Log directory.
#
# === Examples
#
# python::pip { 'flask':
# virtualenv => '/var/www/project1',
# proxy => 'http://proxy.domain.com:3128',
# index => 'http://www.example.com/simple/',
# }
#
# === Authors
#
# Sergey Stankevich
# Fotis Gimian
#
define python::pip (
$pkgname = $name,
$ensure = present,
$virtualenv = 'system',
$url = false,
$owner = 'root',
$group = 'root',
$index = false,
$proxy = false,
$egg = false,
$editable = false,
$environment = [],
$install_args = '',
$uninstall_args = '',
$timeout = 1800,
$log_dir = '/tmp',
$path = ['/usr/local/bin','/usr/bin','/bin', '/usr/sbin'],
) {
$python_provider = getparam(Class['python'], 'provider')
$python_version = getparam(Class['python'], 'version')
# Get SCL exec prefix
# NB: this will not work if you are running puppet from scl enabled shell
$exec_prefix = $python_provider ? {
'scl' => "scl enable ${python_version} -- ",
'rhscl' => "scl enable ${python_version} -- ",
default => '',
}
# Parameter validation
if ! $virtualenv {
fail('python::pip: virtualenv parameter must not be empty')
}
if $virtualenv == 'system' and $owner != 'root' {
fail('python::pip: root user must be used when virtualenv is system')
}
$cwd = $virtualenv ? {
'system' => '/',
default => $virtualenv,
}
validate_absolute_path($cwd)
$log = $virtualenv ? {
'system' => $log_dir,
default => $virtualenv,
}
$pip_env = $virtualenv ? {
'system' => "${exec_prefix}pip",
default => "${exec_prefix}${virtualenv}/bin/pip",
}
$pypi_index = $index ? {
false => '',
default => "--index-url=${index}",
}
$pypi_search_index = $index ? {
false => '',
default => "--index=${index}",
}
$proxy_flag = $proxy ? {
false => '',
default => "--proxy=${proxy}",
}
if $editable == true {
$install_editable = ' -e '
}
else {
$install_editable = ''
}
#TODO: Do more robust argument checking, but below is a start
if ($ensure == absent) and ($install_args != '') {
fail('python::pip cannot provide install_args with ensure => absent')
}
if ($ensure == present) and ($uninstall_args != '') {
fail('python::pip cannot provide uninstall_args with ensure => present')
}
# Check if searching by explicit version.
if $ensure =~ /^((19|20)[0-9][0-9]-(0[1-9]|1[1-2])-([0-2][1-9]|3[0-1])|[0-9]+\.\w+\+?\w*(\.\w+)*)$/ {
$grep_regex = "^${pkgname}==${ensure}\$"
} else {
$grep_regex = $pkgname ? {
/==/ => "^${pkgname}\$",
default => "^${pkgname}==",
}
}
$egg_name = $egg ? {
false => $pkgname,
default => $egg
}
$source = $url ? {
false => $pkgname,
/^(\/|[a-zA-Z]\:)/ => $url,
/^(git\+|hg\+|bzr\+|svn\+)(http|https|ssh|svn|sftp|ftp|lp)(:\/\/).+$/ => $url,
default => "${url}#egg=${egg_name}",
}
# We need to jump through hoops to make sure we issue the correct pip command
# depending on wheel support and versions.
#
# Pip does not support wheels prior to version 1.4.0
# Pip wheels require setuptools/distribute > 0.8
# Python 2.6 and older does not support setuptools/distribute > 0.8
# Pip >= 1.5 tries to use wheels by default, even if wheel package is not
# installed, in this case the --no-use-wheel flag needs to be passed
# Versions prior to 1.5 don't support the --no-use-wheel flag
#
# To check for this we test for wheel parameter using help and then using
# version, this makes sure we only use wheels if they are supported and
# installed
# Explicit version out of VCS when PIP supported URL is provided
if $source =~ /^(git\+|hg\+|bzr\+|svn\+)(http|https|ssh|svn|sftp|ftp|lp)(:\/\/).+$/ {
if $ensure != present and $ensure != latest {
exec { "pip_install_${name}":
command => "${pip_env} wheel --help > /dev/null 2>&1 && { ${pip_env} wheel --version > /dev/null 2>&1 || wheel_support_flag='--no-use-wheel'; } ; { ${pip_env} --log ${log}/pip.log install ${install_args} \$wheel_support_flag ${pypi_index} ${proxy_flag} ${install_args} ${install_editable} ${source}@${ensure}#egg=${egg_name} || ${pip_env} --log ${log}/pip.log install ${install_args} ${pypi_index} ${proxy_flag} ${install_args} ${install_editable} ${source}@${ensure}#egg=${egg_name} ;}",
unless => "${pip_env} freeze | grep -i -e ${grep_regex}",
user => $owner,
group => $group,
cwd => $cwd,
environment => $environment,
timeout => $timeout,
path => $path,
}
} else {
exec { "pip_install_${name}":
command => "${pip_env} wheel --help > /dev/null 2>&1 && { ${pip_env} wheel --version > /dev/null 2>&1 || wheel_support_flag='--no-use-wheel'; } ; { ${pip_env} --log ${log}/pip.log install ${install_args} \$wheel_support_flag ${pypi_index} ${proxy_flag} ${install_args} ${install_editable} ${source} || ${pip_env} --log ${log}/pip.log install ${install_args} ${pypi_index} ${proxy_flag} ${install_args} ${install_editable} ${source} ;}",
unless => "${pip_env} freeze | grep -i -e ${grep_regex}",
user => $owner,
group => $group,
cwd => $cwd,
environment => $environment,
timeout => $timeout,
path => $path,
}
}
} else {
case $ensure {
/^((19|20)[0-9][0-9]-(0[1-9]|1[1-2])-([0-2][1-9]|3[0-1])|[0-9]+\.\w+\+?\w*(\.\w+)*)$/: {
# Version formats as per http://guide.python-distribute.org/specification.html#standard-versioning-schemes
# Explicit version.
exec { "pip_install_${name}":
command => "${pip_env} wheel --help > /dev/null 2>&1 && { ${pip_env} wheel --version > /dev/null 2>&1 || wheel_support_flag='--no-use-wheel'; } ; { ${pip_env} --log ${log}/pip.log install ${install_args} \$wheel_support_flag ${pypi_index} ${proxy_flag} ${install_args} ${install_editable} ${source}==${ensure} || ${pip_env} --log ${log}/pip.log install ${install_args} ${pypi_index} ${proxy_flag} ${install_args} ${install_editable} ${source}==${ensure} ;}",
unless => "${pip_env} freeze | grep -i -e ${grep_regex} || ${pip_env} list | sed -e 's/[ ]\\+/==/' -e 's/[()]//g' | grep -i -e ${grep_regex}",
user => $owner,
group => $group,
cwd => $cwd,
environment => $environment,
timeout => $timeout,
path => $path,
}
}
#
present: {
# Whatever version is available.
exec { "pip_install_${name}":
command => "${pip_env} wheel --help > /dev/null 2>&1 && { ${pip_env} wheel --version > /dev/null 2>&1 || wheel_support_flag='--no-use-wheel'; } ; { ${pip_env} --log ${log}/pip.log install \$wheel_support_flag ${pypi_index} ${proxy_flag} ${install_args} ${install_editable} ${source} || ${pip_env} --log ${log}/pip.log install ${pypi_index} ${proxy_flag} ${install_args} ${install_editable} ${source} ;}",
unless => "${pip_env} freeze | grep -i -e ${grep_regex} || ${pip_env} list | sed -e 's/[ ]\\+/==/' -e 's/[()]//g' | grep -i -e ${grep_regex}",
user => $owner,
group => $group,
cwd => $cwd,
environment => $environment,
timeout => $timeout,
path => $path,
}
}
latest: {
# Latest version.
exec { "pip_install_${name}":
command => "${pip_env} wheel --help > /dev/null 2>&1 && { ${pip_env} wheel --version > /dev/null 2>&1 || wheel_support_flag='--no-use-wheel'; } ; { ${pip_env} --log ${log}/pip.log install --upgrade \$wheel_support_flag ${pypi_index} ${proxy_flag} ${install_args} ${install_editable} ${source} || ${pip_env} --log ${log}/pip.log install --upgrade ${pypi_index} ${proxy_flag} ${install_args} ${install_editable} ${source} ;}",
unless => "${pip_env} search ${pypi_search_index} ${proxy_flag} ${source} | grep -i INSTALLED.*latest",
user => $owner,
group => $group,
cwd => $cwd,
environment => $environment,
timeout => $timeout,
path => $path,
}
}
default: {
# Anti-action, uninstall.
exec { "pip_uninstall_${name}":
command => "echo y | ${pip_env} uninstall ${uninstall_args} ${proxy_flag} ${name}",
onlyif => "${pip_env} freeze | grep -i -e ${grep_regex}",
user => $owner,
group => $group,
cwd => $cwd,
environment => $environment,
timeout => $timeout,
path => $path,
}
}
}
}
}

104
modules/utilities/unix/languages/python2/python/manifests/pyvenv.pp Normal file
View File

@@ -0,0 +1,104 @@
# == Define: python::pyvenv
#
# Create a Python3 virtualenv using pyvenv.
#
# === Parameters
#
# [*ensure*]
# present|absent. Default: present
#
# [*version*]
# Python version to use. Default: system default
#
# [*systempkgs*]
# Copy system site-packages into virtualenv. Default: don't
#
# [*venv_dir*]
# Directory to install virtualenv to. Default: $name
#
# [*owner*]
# The owner of the virtualenv being manipulated. Default: root
#
# [*group*]
# The group relating to the virtualenv being manipulated. Default: root
#
# [*mode*]
# Optionally specify directory mode. Default: 0755
#
# [*path*]
# Specifies the PATH variable. Default: [ '/bin', '/usr/bin', '/usr/sbin' ]
# [*environment*]
# Optionally specify environment variables for pyvenv
#
# === Examples
#
# python::venv { '/var/www/project1':
# ensure => present,
# version => 'system',
# systempkgs => true,
# }
#
# === Authors
#
# Sergey Stankevich
# Ashley Penney
# Marc Fournier
# Fotis Gimian
# Seth Cleveland
#
define python::pyvenv (
$ensure = present,
$version = 'system',
$systempkgs = false,
$venv_dir = $name,
$owner = 'root',
$group = 'root',
$mode = '0755',
$path = [ '/bin', '/usr/bin', '/usr/sbin', '/usr/local/bin' ],
$environment = [],
) {
include ::python
if $ensure == 'present' {
$virtualenv_cmd = $version ? {
'system' => "${python::exec_prefix}pyvenv",
default => "${python::exec_prefix}pyvenv-${version}",
}
if ( $systempkgs == true ) {
$system_pkgs_flag = '--system-site-packages'
} else {
$system_pkgs_flag = ''
}
file { $venv_dir:
ensure => directory,
owner => $owner,
group => $group,
mode => $mode
}
exec { "python_virtualenv_${venv_dir}":
command => "${virtualenv_cmd} --clear ${system_pkgs_flag} ${venv_dir}",
user => $owner,
creates => "${venv_dir}/bin/activate",
path => $path,
cwd => '/tmp',
environment => $environment,
unless => "grep '^[\\t ]*VIRTUAL_ENV=[\\\\'\\\"]*${venv_dir}[\\\"\\\\'][\\t ]*$' ${venv_dir}/bin/activate", #Unless activate exists and VIRTUAL_ENV is correct we re-create the virtualenv
require => File[$venv_dir],
}
} elsif $ensure == 'absent' {
file { $venv_dir:
ensure => absent,
force => true,
recurse => true,
purge => true,
}
} else {
fail( "Illegal ensure value: ${ensure}. Expected (present or absent)")
}
}

141
modules/utilities/unix/languages/python2/python/manifests/requirements.pp Normal file
View File

@@ -0,0 +1,141 @@
# == Define: python::requirements
#
# Installs and manages Python packages from requirements file.
#
# === Parameters
#
# [*requirements*]
# Path to the requirements file. Defaults to the resource name
#
# [*virtualenv*]
# virtualenv to run pip in. Default: system-wide
#
# [*owner*]
# The owner of the virtualenv being manipulated. Default: root
#
# [*group*]
# The group relating to the virtualenv being manipulated. Default: root
#
# [*proxy*]
# Proxy server to use for outbound connections. Default: none
#
# [*src*]
# Pip --src parameter; if the requirements file contains --editable resources,
# this parameter specifies where they will be installed. See the pip
# documentation for more. Default: none (i.e. use the pip default).
#
# [*environment*]
# Additional environment variables required to install the packages. Default: none
#
# [*forceupdate*]
# Run a pip install requirements even if we don't receive an event from the
# requirements file - Useful for when the requirements file is written as part of a
# resource other than file (E.g vcsrepo)
#
# [*cwd*]
# The directory from which to run the "pip install" command. Default: undef
#
# [*extra_pip_args*]
# Extra arguments to pass to pip after the requirements file
#
# [*manage_requirements*]
# Create the requirements file if it doesn't exist. Default: true
#
# [*fix_requirements_owner*]
# Change owner and group of requirements file. Default: true
#
# [*log_dir*]
# String. Log directory.
#
# [*timeout*]
# The maximum time in seconds the "pip install" command should take. Default: 1800
#
# === Examples
#
# python::requirements { '/var/www/project1/requirements.txt':
# virtualenv => '/var/www/project1',
# proxy => 'http://proxy.domain.com:3128',
# }
#
# === Authors
#
# Sergey Stankevich
# Ashley Penney
# Fotis Gimian
#
define python::requirements (
$requirements = $name,
$virtualenv = 'system',
$owner = 'root',
$group = 'root',
$proxy = false,
$src = false,
$environment = [],
$forceupdate = false,
$cwd = undef,
$extra_pip_args = '',
$manage_requirements = true,
$fix_requirements_owner = true,
$log_dir = '/tmp',
$timeout = 1800,
) {
include ::python
if $virtualenv == 'system' and ($owner != 'root' or $group != 'root') {
fail('python::pip: root user must be used when virtualenv is system')
}
if $fix_requirements_owner {
$owner_real = $owner
$group_real = $group
} else {
$owner_real = undef
$group_real = undef
}
$log = $virtualenv ? {
'system' => $log_dir,
default => $virtualenv,
}
$pip_env = $virtualenv ? {
'system' => "${::python::exec_prefix} pip",
default => "${::python::exec_prefix} ${virtualenv}/bin/pip",
}
$proxy_flag = $proxy ? {
false => '',
default => "--proxy=${proxy}",
}
$src_flag = $src ? {
false => '',
default => "--src=${src}",
}
# This will ensure multiple python::virtualenv definitions can share the
# the same requirements file.
if !defined(File[$requirements]) and $manage_requirements == true {
file { $requirements:
ensure => present,
mode => '0644',
owner => $owner_real,
group => $group_real,
audit => content,
replace => false,
content => '# Puppet will install and/or update pip packages listed here',
}
}
exec { "python_requirements${name}":
provider => shell,
command => "${pip_env} --log ${log}/pip.log install ${proxy_flag} ${src_flag} -r ${requirements} ${extra_pip_args}",
refreshonly => !$forceupdate,
timeout => $timeout,
cwd => $cwd,
user => $owner,
subscribe => File[$requirements],
environment => $environment,
}
}

208
modules/utilities/unix/languages/python2/python/manifests/virtualenv.pp Normal file
View File

@@ -0,0 +1,208 @@
# == Define: python::virtualenv
#
# Creates Python virtualenv.
#
# === Parameters
#
# [*ensure*]
# present|absent. Default: present
#
# [*version*]
# Python version to use. Default: system default
#
# [*requirements*]
# Path to pip requirements.txt file. Default: none
#
# [*systempkgs*]
# Copy system site-packages into virtualenv. Default: don't
# If virtualenv version < 1.7 this flag has no effect since
# [*venv_dir*]
# Directory to install virtualenv to. Default: $name
#
# [*distribute*]
# Include distribute in the virtualenv. Default: true
#
# [*index*]
# Base URL of Python package index. Default: none (http://pypi.python.org/simple/)
#
# [*owner*]
# The owner of the virtualenv being manipulated. Default: root
#
# [*group*]
# The group relating to the virtualenv being manipulated. Default: root
#
# [*mode*]
# Optionally specify directory mode. Default: 0755
#
# [*proxy*]
# Proxy server to use for outbound connections. Default: none
#
# [*environment*]
# Additional environment variables required to install the packages. Default: none
#
# [*path*]
# Specifies the PATH variable. Default: [ '/bin', '/usr/bin', '/usr/sbin' ]
#
# [*cwd*]
# The directory from which to run the "pip install" command. Default: undef
#
# [*timeout*]
# The maximum time in seconds the "pip install" command should take. Default: 1800
#
# [*pip_args*]
# Arguments to pass to pip during initialization. Default: blank
#
# [*extra_pip_args*]
# Extra arguments to pass to pip after requirements file. Default: blank
#
# === Examples
#
# python::virtualenv { '/var/www/project1':
# ensure => present,
# version => 'system',
# requirements => '/var/www/project1/requirements.txt',
# proxy => 'http://proxy.domain.com:3128',
# systempkgs => true,
# index => 'http://www.example.com/simple/',
# }
#
# === Authors
#
# Sergey Stankevich
# Shiva Poudel
#
define python::virtualenv (
$ensure = present,
$version = 'system',
$requirements = false,
$systempkgs = false,
$venv_dir = $name,
$distribute = true,
$index = false,
$owner = 'root',
$group = 'root',
$mode = '0755',
$proxy = false,
$environment = [],
$path = [ '/bin', '/usr/bin', '/usr/sbin', '/usr/local/bin' ],
$cwd = undef,
$timeout = 1800,
$pip_args = '',
$extra_pip_args = '',
$virtualenv = undef
) {
include ::python
if $ensure == 'present' {
$python = $version ? {
'system' => 'python',
'pypy' => 'pypy',
default => "python${version}",
}
if $virtualenv == undef {
$used_virtualenv = 'virtualenv'
} else {
$used_virtualenv = $virtualenv
}
$proxy_flag = $proxy ? {
false => '',
default => "--proxy=${proxy}",
}
$proxy_command = $proxy ? {
false => '',
default => "&& export http_proxy=${proxy}",
}
# Virtualenv versions prior to 1.7 do not support the
# --system-site-packages flag, default off for prior versions
# Prior to version 1.7 the default was equal to --system-site-packages
# and the flag --no-site-packages had to be passed to do the opposite
$_virtualenv_version = getvar('virtualenv_version') ? {
/.*/ => getvar('virtualenv_version'),
default => '',
}
if (( versioncmp($_virtualenv_version,'1.7') > 0 ) and ( $systempkgs == true )) {
$system_pkgs_flag = '--system-site-packages'
} elsif (( versioncmp($_virtualenv_version,'1.7') < 0 ) and ( $systempkgs == false )) {
$system_pkgs_flag = '--no-site-packages'
} else {
$system_pkgs_flag = $systempkgs ? {
true => '--system-site-packages',
false => '--no-site-packages',
default => fail('Invalid value for systempkgs. Boolean value is expected')
}
}
$distribute_pkg = $distribute ? {
true => 'distribute',
default => 'setuptools',
}
$pypi_index = $index ? {
false => '',
default => "-i ${index}",
}
# Python 2.6 and older does not support setuptools/distribute > 0.8 which
# is required for pip wheel support, pip therefor requires --no-use-wheel flag
# if the # pip version is more recent than 1.4.1 but using an old python or
# setuputils/distribute version
# To check for this we test for wheel parameter using help and then using
# version, this makes sure we only use wheels if they are supported
file { $venv_dir:
ensure => directory,
owner => $owner,
group => $group,
mode => $mode
}
$virtualenv_cmd = "${python::exec_prefix}${used_virtualenv}"
$pip_cmd = "${python::exec_prefix}${venv_dir}/bin/pip"
$pip_flags = "${pypi_index} ${proxy_flag} ${pip_args}"
exec { "python_virtualenv_${venv_dir}":
command => "true ${proxy_command} && ${virtualenv_cmd} ${system_pkgs_flag} -p ${python} ${venv_dir} && ${pip_cmd} --log ${venv_dir}/pip.log install ${pip_flags} --upgrade pip && ${pip_cmd} install ${pip_flags} --upgrade ${distribute_pkg}",
user => $owner,
creates => "${venv_dir}/bin/activate",
path => $path,
cwd => '/tmp',
environment => $environment,
unless => "grep '^[\\t ]*VIRTUAL_ENV=[\\\\'\\\"]*${venv_dir}[\\\"\\\\'][\\t ]*$' ${venv_dir}/bin/activate", #Unless activate exists and VIRTUAL_ENV is correct we re-create the virtualenv
require => File[$venv_dir],
}
if $requirements {
exec { "python_requirements_initial_install_${requirements}_${venv_dir}":
command => "${pip_cmd} --log ${venv_dir}/pip.log install ${pypi_index} ${proxy_flag} --no-binary :all: -r ${requirements} ${extra_pip_args}",
refreshonly => true,
timeout => $timeout,
user => $owner,
subscribe => Exec["python_virtualenv_${venv_dir}"],
environment => $environment,
cwd => $cwd
}
python::requirements { "${requirements}_${venv_dir}":
requirements => $requirements,
virtualenv => $venv_dir,
proxy => $proxy,
owner => $owner,
group => $group,
cwd => $cwd,
require => Exec["python_virtualenv_${venv_dir}"],
extra_pip_args => $extra_pip_args,
}
}
} elsif $ensure == 'absent' {
file { $venv_dir:
ensure => absent,
force => true,
recurse => true,
purge => true,
}
}
}