OSDN Git Service

add cdh::yum_mirror recipe.
authorwhitestar <whitestar@gaea.test>
Wed, 2 Oct 2013 13:01:05 +0000 (22:01 +0900)
committerwhitestar <whitestar@gaea.test>
Wed, 2 Oct 2013 13:01:05 +0000 (22:01 +0900)
18 files changed:
cookbooks/cdh/attributes/default.rb
cookbooks/cdh/recipes/yum_mirror.rb [new file with mode: 0644]
cookbooks/cdh/templates/default/etc/apache2/conf.d/cdh-mirror [new file with mode: 0644]
cookbooks/cdh/templates/default/etc/cron.d/cdh-mirror [new file with mode: 0644]
cookbooks/cdh/templates/default/usr/local/bin/cdh-mirror [new file with mode: 0644]
cookbooks/hdp/attributes/default.rb
cookbooks/hdp/recipes/yum_mirror.rb
cookbooks/hdp/templates/default/etc/apache2/conf.d/hdp-mirror
cookbooks/yum_utils/attributes/default.rb
cookbooks/yum_utils/libraries/repo_utils.rb
cookbooks/yum_utils/recipes/mirror.rb
cookbooks/yum_utils/recipes/reposync_mirror.rb [new file with mode: 0644]
cookbooks/yum_utils/templates/default/etc/apache2/conf.d/yum-reposync-mirror [new file with mode: 0644]
cookbooks/yum_utils/templates/default/etc/cron.d/yum-reposync-mirror [new file with mode: 0644]
cookbooks/yum_utils/templates/default/usr/local/bin/yum-reposync-mirror [new file with mode: 0644]
roles/cdh-mirror.rb [new file with mode: 0644]
roles/grid-ftp.rb
roles/yum-reposync-mirror.rb [new file with mode: 0644]

index 2b330d8..a5ba62d 100644 (file)
@@ -25,5 +25,16 @@ default['cdh']['yum_repo']['update'] = false
 default['cdh']['yum_mirror']['user'] = 'yum-mirror'
 default['cdh']['yum_mirror']['cron_period'] = '#0 4   * * *'  # default: inactive
 default['cdh']['yum_mirror']['base_path'] = '/var/spool/cdh-mirror'
+default['cdh']['yum_mirror']['yum_conf'] = value_for_platform_family(
+  'debian' => '/etc/yum/yum.conf',
+  'rhel' => '/etc/yum.conf'
+)
+default['cdh']['yum_mirror']['repos_dir'] = value_for_platform_family(
+  'debian' => '/etc/yum/repos.d',
+  'rhel' => '/etc/yum.repos.d'
+)
+default['cdh']['yum_mirror']['repo_ids'] = []
+default['cdh']['yum_mirror']['arch'] = ''
+default['cdh']['yum_mirror']['url_alias_with_authority_part'] = true
 #default['cdh'][''] = 
 
diff --git a/cookbooks/cdh/recipes/yum_mirror.rb b/cookbooks/cdh/recipes/yum_mirror.rb
new file mode 100644 (file)
index 0000000..192bacb
--- /dev/null
@@ -0,0 +1,88 @@
+#
+# Cookbook Name:: cdh
+# Recipe:: yum_mirror
+#
+# Copyright 2013, whitestar
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+::Chef::Recipe.send(:include, Commons::PackageUtils)
+::Chef::Recipe.send(:include, YumUtils::RepoUtils)
+
+prefix = 'cdh'
+yum_mirror = node['cdh']['yum_mirror']
+
+reposync_sources, url_aliases = reposync_base_setup(
+  yum_mirror['user'],
+  yum_mirror['base_path'],
+  yum_mirror['yum_conf'],
+  yum_mirror['repos_dir'],
+  yum_mirror['repo_ids'],
+  yum_mirror['arch'],
+  yum_mirror['url_alias_with_authority_part']
+)
+
+template "/usr/local/bin/#{prefix}-mirror" do
+  source "usr/local/bin/#{prefix}-mirror"
+  owner 'root'
+  group 'root'
+  mode '0755'
+  variables({
+    :reposync_sources => reposync_sources
+  })
+end
+
+pkg = get_cron_pkg_name
+resources(:package => pkg) rescue package pkg do
+  action :install
+end
+
+template "/etc/cron.d/#{prefix}-mirror" do
+  source "etc/cron.d/#{prefix}-mirror"
+  owner 'root'
+  group 'root'
+  mode '0644'
+end
+
+httpd_service = get_httpd_pkg_name
+httpd_conf_path = nil
+
+case node[:platform_family]
+when 'debian'
+  httpd_conf_path = "/etc/#{httpd_service}/conf.d/#{prefix}-mirror"
+when 'rhel'
+  httpd_conf_path = "/etc/#{httpd_service}/conf.d/#{prefix}-mirror.conf"
+end
+
+pkg = httpd_service
+resources(:package => pkg) rescue package pkg do
+  action :install
+end
+
+resources(:service => httpd_service) rescue service httpd_service do
+  action [:enable, :start]
+  supports :status => true, :restart => true, :reload => true
+end
+
+template httpd_conf_path do
+  source "etc/apache2/conf.d/#{prefix}-mirror"
+  owner 'root'
+  group 'root'
+  mode '0644'
+  variables({
+    :url_aliases => url_aliases
+  })
+  notifies :restart, "service[#{httpd_service}]"
+end
+
diff --git a/cookbooks/cdh/templates/default/etc/apache2/conf.d/cdh-mirror b/cookbooks/cdh/templates/default/etc/apache2/conf.d/cdh-mirror
new file mode 100644 (file)
index 0000000..4de77bd
--- /dev/null
@@ -0,0 +1,33 @@
+#
+# Copyright 2013, whitestar
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+<%
+@url_aliases.each do |url_path, real_path|
+-%>
+Alias <%= url_path %> <%= real_path %> 
+<Directory <%= real_path %>>
+  Options Indexes SymLinksIfOwnerMatch FollowSymLinks
+  IndexOptions NameWidth=* +SuppressDescription
+  AllowOverride FileInfo Indexes
+  Order Deny,Allow
+  Allow from all
+  ErrorDocument 404 default
+</Directory>
+
+<%
+end
+-%>
+
diff --git a/cookbooks/cdh/templates/default/etc/cron.d/cdh-mirror b/cookbooks/cdh/templates/default/etc/cron.d/cdh-mirror
new file mode 100644 (file)
index 0000000..ad34607
--- /dev/null
@@ -0,0 +1,19 @@
+#
+# Regular cron jobs for the yum-mirror
+#
+# Copyright 2013, whitestar
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+<%= node['cdh']['yum_mirror']['cron_period'] %>        <%= node['cdh']['yum_mirror']['user'] %>        /usr/local/bin/cdh-mirror > <%= node['cdh']['yum_mirror']['base_path'] %>/var/cron.log 2>&1
diff --git a/cookbooks/cdh/templates/default/usr/local/bin/cdh-mirror b/cookbooks/cdh/templates/default/usr/local/bin/cdh-mirror
new file mode 100644 (file)
index 0000000..d342a3c
--- /dev/null
@@ -0,0 +1,29 @@
+#!/bin/sh
+#
+# Copyright 2013, whitestar
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+<%
+@reposync_sources.each do |source|
+-%>
+echo -n 'Begin time: '; date
+<%= source['reposync_command'] %>
+<%= source['createrepo_command'] %>
+echo -n 'End time: '; date
+
+<%
+end
+-%>
+
index 9469ed3..cf0dc54 100644 (file)
@@ -37,5 +37,6 @@ default['hdp']['yum_mirror']['repos_dir'] = value_for_platform_family(
   'rhel' => '/etc/yum.repos.d'
 )
 default['hdp']['yum_mirror']['repo_ids'] = []
+default['hdp']['yum_mirror']['url_alias_with_authority_part'] = true
 #default['hdp'][''] = 
 
index 93a2db0..da5e550 100644 (file)
 ::Chef::Recipe.send(:include, Commons::PackageUtils)
 ::Chef::Recipe.send(:include, YumUtils::RepoUtils)
 
+prefix = 'hdp'
+yum_mirror = node['hdp']['yum_mirror']
 # HDP supports only x86_64 architecture.
 arch = 'x86_64'
-mirror_user = node['hdp']['yum_mirror']['user']
-base_path = node['hdp']['yum_mirror']['base_path']
 
-%w{
-  yum-utils
-  createrepo
-}.each {|pkg|
-  resources(:package => pkg) rescue package pkg do
-    action :install
-  end
-}
-
-resources(:group => mirror_user) rescue group mirror_user do
-  system true
-  action :create
-end
-
-resources(:user => mirror_user) rescue user mirror_user do
-  gid mirror_user
-  system true
-  home base_path
-  shell '/bin/sh'
-  password nil
-  action :create
-end
-
-httpd_service = get_httpd_pkg_name
-httpd_conf_path = nil
-
-case node[:platform_family]
-when 'debian'
-  httpd_conf_path = "/etc/#{httpd_service}/conf.d/hdp-mirror"
-when 'rhel'
-  httpd_conf_path = "/etc/#{httpd_service}/conf.d/hdp-mirror.conf"
-end
-
-pkg = get_cron_pkg_name
-resources(:package => pkg) rescue package pkg do
-  action :install
-end
-
-[
-  '',
-  'mirror',
-  'var',
-].each {|subdir|
-  directory "#{base_path}/#{subdir}" do
-    owner mirror_user
-    group mirror_user
-    mode '0755'
-    recursive true
-  end
-}
-
-repos = parse_repo_files(node['hdp']['yum_mirror']['repos_dir'])
-
-reposync_sources = []
-http_aliases = {}
-node['hdp']['yum_mirror']['repo_ids'].each {|repo_id|
-  unless repos.has_key?(repo_id) then
-    Chef::Application.fatal!("Yum repository definition (id: #{repo_id}) not found!")
-  end
-
-  repo = repos[repo_id]
-  ctx_path = repo['baseurl'].gsub(/^http:\/\/[^\/]*\//, '')
-
-  local_path = "#{base_path}/mirror/#{ctx_path}"
-  if local_path.end_with?('/') then
-    local_path.chop!
-  end
-  local_parent_path = File.dirname(local_path)
-  actual_repo_path = "#{local_parent_path}/#{repo_id}"
-  local_gpgkey_path = "#{base_path}/mirror/#{repo['gpgkey'].gsub(/^http:\/\/[^\/]*\//, '')}"
-
-  [
-    local_parent_path,
-    actual_repo_path,
-    File.dirname(local_gpgkey_path)
-  ].each {|dir|
-    resources(:directory => dir) rescue directory dir do
-      owner mirror_user
-      group mirror_user
-      mode '0755'
-      recursive true
-    end
-  }
-
-  link local_path do
-    to actual_repo_path
-  end
-
-  resources(:remote_file => local_gpgkey_path) rescue remote_file local_gpgkey_path do
-    source repo['gpgkey']
-    owner mirror_user
-    group mirror_user
-    mode '0644'
-  end
-
-  reposync_sources.push(
-    {
-      'id' => repo_id,
-      'local_path' => local_path,
-      'reposync_command' => "reposync -c #{node['hdp']['yum_mirror']['yum_conf']} -a #{arch} -r #{repo_id} -p #{local_parent_path}",
-      'createrepo_command' => "createrepo #{local_path}"
-    }
-  )
-
-  ctx_path_top = ctx_path.split('/')[0]
-  http_aliases["/#{ctx_path_top}"] = "#{base_path}/mirror/#{ctx_path_top}"
-}
-
-template '/usr/local/bin/hdp-mirror' do
-  source 'usr/local/bin/hdp-mirror'
+reposync_sources, url_aliases = reposync_base_setup(
+  yum_mirror['user'],
+  yum_mirror['base_path'],
+  yum_mirror['yum_conf'],
+  yum_mirror['repos_dir'],
+  yum_mirror['repo_ids'],
+  arch,
+  yum_mirror['url_alias_with_authority_part']
+)
+
+template "/usr/local/bin/#{prefix}-mirror" do
+  source "usr/local/bin/#{prefix}-mirror"
   owner 'root'
   group 'root'
   mode '0755'
@@ -143,13 +45,28 @@ template '/usr/local/bin/hdp-mirror' do
   })
 end
 
-template '/etc/cron.d/hdp-mirror' do
-  source 'etc/cron.d/hdp-mirror'
+pkg = get_cron_pkg_name
+resources(:package => pkg) rescue package pkg do
+  action :install
+end
+
+template "/etc/cron.d/#{prefix}-mirror" do
+  source "etc/cron.d/#{prefix}-mirror"
   owner 'root'
   group 'root'
   mode '0644'
 end
 
+httpd_service = get_httpd_pkg_name
+httpd_conf_path = nil
+
+case node[:platform_family]
+when 'debian'
+  httpd_conf_path = "/etc/#{httpd_service}/conf.d/#{prefix}-mirror"
+when 'rhel'
+  httpd_conf_path = "/etc/#{httpd_service}/conf.d/#{prefix}-mirror.conf"
+end
+
 pkg = httpd_service
 resources(:package => pkg) rescue package pkg do
   action :install
@@ -161,12 +78,12 @@ resources(:service => httpd_service) rescue service httpd_service do
 end
 
 template httpd_conf_path do
-  source 'etc/apache2/conf.d/hdp-mirror'
+  source "etc/apache2/conf.d/#{prefix}-mirror"
   owner 'root'
   group 'root'
   mode '0644'
   variables({
-    :http_aliases => http_aliases
+    :url_aliases => url_aliases
   })
   notifies :restart, "service[#{httpd_service}]"
 end
index e245534..4de77bd 100644 (file)
@@ -15,7 +15,7 @@
 #
 
 <%
-@http_aliases.each do |url_path, real_path|
+@url_aliases.each do |url_path, real_path|
 -%>
 Alias <%= url_path %> <%= real_path %> 
 <Directory <%= real_path %>>
index 1d20c97..0613315 100644 (file)
@@ -35,6 +35,7 @@ default['yum_utils']['repos']['epel'] = {
   'gpgkey' => 'file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-6'
 }
 
+# Repository mirroring by rsync
 default['yum_utils']['mirror']['user'] = 'yum-mirror'
 default['yum_utils']['mirror']['cron_period'] = '#0 2  * * *'  # default: inactive
 default['yum_utils']['mirror']['base_path'] = '/var/spool/yum-mirror'
@@ -59,3 +60,19 @@ default['yum_utils']['mirror']['rsync_sources'] = [
 =end
 ]
 
+# Repository mirroring by reposync
+default['yum_utils']['reposync_mirror']['user'] = 'yum-mirror'
+default['yum_utils']['reposync_mirror']['cron_period'] = '#0 3   * * *'  # default: inactive
+default['yum_utils']['reposync_mirror']['base_path'] = '/var/spool/yum-reposync-mirror'
+default['yum_utils']['reposync_mirror']['yum_conf'] = value_for_platform_family(
+  'debian' => '/etc/yum/yum.conf',
+  'rhel' => '/etc/yum.conf'
+)
+default['yum_utils']['reposync_mirror']['repos_dir'] = value_for_platform_family(
+  'debian' => '/etc/yum/repos.d',
+  'rhel' => '/etc/yum.repos.d'
+)
+default['yum_utils']['reposync_mirror']['repo_ids'] = []
+default['yum_utils']['reposync_mirror']['arch'] = ''
+default['yum_utils']['reposync_mirror']['url_alias_with_authority_part'] = true
+
index 06b5360..7add7b3 100644 (file)
 
 module YumUtils
   module RepoUtils
+    def create_mirror_user(name, home_dir)
+      resources(:group => name) rescue group name do
+        system true
+        action :create
+      end
+      
+      resources(:user => name) rescue user name do
+        gid name
+        system true
+        home home_dir
+        shell '/bin/sh'
+        password nil
+        action :create
+      end
+    end
+
+
+    def create_mirror_dirs(mirror_user, base_path)
+      [
+        '',
+        'mirror',
+        'var',
+      ].each {|subdir|
+        directory "#{base_path}/#{subdir}" do
+          owner mirror_user
+          group mirror_user
+          mode '0755'
+          recursive true
+        end
+      }
+    end
+
+
     def parse_repo_files(repos_dir)
       repos = {}
       current_repo = nil
@@ -37,7 +70,7 @@ module YumUtils
               repos[repo_id] = current_repo
             else
               prop = line.split('=')
-              current_repo[prop[0]] = prop[1]
+              current_repo[prop[0].strip] = prop[1].strip
             end
           }
         }
@@ -45,6 +78,104 @@ module YumUtils
     
       return repos
     end
+
+
+    def install_reposync()
+      %w{
+        yum-utils
+        createrepo
+      }.each {|pkg|
+        resources(:package => pkg) rescue package pkg do
+          action :install
+        end
+      }
+    end
+
+
+    def reposync_base_setup(
+      mirror_user,
+      base_path,
+      yum_conf_path,
+      repos_dir,
+      repo_ids,
+      arch,
+      url_alias_with_authority_part = true)
+
+      install_reposync()
+      create_mirror_user(mirror_user, base_path)
+      create_mirror_dirs(mirror_user, base_path)
+      
+      repos = parse_repo_files(repos_dir)
+      
+      reposync_sources = []
+      url_aliases = {}
+      repo_ids.each {|repo_id|
+        unless repos.has_key?(repo_id) then
+          Chef::Application.fatal!("Yum repository definition (id: #{repo_id}) not found!")
+        end
+      
+        repo = repos[repo_id]
+        ctx_path = repo['baseurl'].gsub(/^http:\/\//, '')
+        gpgkey_ctx_path = repo['gpgkey'].gsub(/^http:\/\//, '')
+      
+        local_path = "#{base_path}/mirror/#{ctx_path}"
+        if local_path.end_with?('/') then
+          local_path.chop!
+        end
+        local_parent_path = File.dirname(local_path)
+        actual_repo_path = "#{local_parent_path}/#{repo_id}"
+        local_gpgkey_path = "#{base_path}/mirror/#{gpgkey_ctx_path}"
+      
+        [
+          local_parent_path,
+          actual_repo_path,
+          File.dirname(local_gpgkey_path)
+        ].each {|dir|
+          resources(:directory => dir) rescue directory dir do
+            owner mirror_user
+            group mirror_user
+            mode '0755'
+            recursive true
+          end
+        }
+      
+        link local_path do
+          to actual_repo_path
+        end
+      
+        resources(:remote_file => local_gpgkey_path) rescue remote_file local_gpgkey_path do
+          source repo['gpgkey']
+          owner mirror_user
+          group mirror_user
+          mode '0644'
+        end
+      
+        reposync_sources.push(
+          {
+            'id' => repo_id,
+            'local_path' => local_path,
+            'reposync_command' => "reposync -c #{yum_conf_path} -a #{arch} -r #{repo_id} -p #{local_parent_path}",
+            'createrepo_command' => "createrepo #{local_path}"
+          }
+        )
+      
+        ctx_path_elms= ctx_path.split('/')
+        gpgkey_ctx_path_elms = gpgkey_ctx_path.split('/')
+        if url_alias_with_authority_part then
+          url_aliases["/#{ctx_path_elms[0]}"] \
+            = "#{base_path}/mirror/#{ctx_path_elms[0]}"
+          url_aliases["/#{gpgkey_ctx_path_elms[0]}"] \
+            = "#{base_path}/mirror/#{gpgkey_ctx_path_elms[0]}"
+        else
+          url_aliases["/#{ctx_path_elms[1]}"] \
+            = "#{base_path}/mirror/#{ctx_path_elms[0]}/#{ctx_path_elms[1]}"
+          url_aliases["/#{gpgkey_ctx_path_elms[1]}"] \
+            = "#{base_path}/mirror/#{gpgkey_ctx_path_elms[0]}/#{gpgkey_ctx_path_elms[1]}"
+        end
+      }
+      
+      return [reposync_sources, url_aliases]
+    end
   end
 end
 
index 1509150..f115f20 100644 (file)
 #
 
 ::Chef::Recipe.send(:include, Commons::PackageUtils)
+::Chef::Recipe.send(:include, YumUtils::RepoUtils)
 
 mirror_user = node['yum_utils']['mirror']['user']
 base_path = node['yum_utils']['mirror']['base_path']
 
-resources(:group => mirror_user) rescue group mirror_user do
-  system true
-  action :create
-end
-
-resources(:user => mirror_user) rescue user mirror_user do
-  gid mirror_user
-  system true
-  home base_path
-  shell '/bin/sh'
-  password nil
-  action :create
-end
+create_mirror_user(mirror_user, base_path)
+create_mirror_dirs(mirror_user, base_path)
 
 httpd_service = get_httpd_pkg_name
 httpd_conf_path = nil
@@ -56,19 +46,6 @@ resources(:package => pkg) rescue package pkg do
   action :install
 end
 
-[
-  '',
-  'mirror',
-  'var',
-].each {|subdir|
-  directory "#{base_path}/#{subdir}" do
-    owner mirror_user
-    group mirror_user
-    mode '0755'
-    recursive true
-  end
-}
-
 rsync_sources = []
 node['yum_utils']['mirror']['rsync_sources'].each {|source|
   local_path = "#{base_path}/mirror/#{source['url'].gsub(/^rsync:\/\//, '')}"
diff --git a/cookbooks/yum_utils/recipes/reposync_mirror.rb b/cookbooks/yum_utils/recipes/reposync_mirror.rb
new file mode 100644 (file)
index 0000000..6771c1b
--- /dev/null
@@ -0,0 +1,88 @@
+#
+# Cookbook Name:: yum_utils
+# Recipe:: reposync_mirror
+#
+# Copyright 2013, whitestar
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+::Chef::Recipe.send(:include, Commons::PackageUtils)
+::Chef::Recipe.send(:include, YumUtils::RepoUtils)
+
+prefix = 'yum-reposync'
+yum_mirror = node['yum_utils']['reposync_mirror']
+
+reposync_sources, url_aliases = reposync_base_setup(
+  yum_mirror['user'],
+  yum_mirror['base_path'],
+  yum_mirror['yum_conf'],
+  yum_mirror['repos_dir'],
+  yum_mirror['repo_ids'],
+  yum_mirror['arch'],
+  yum_mirror['url_alias_with_authority_part']
+)
+
+template "/usr/local/bin/#{prefix}-mirror" do
+  source "usr/local/bin/#{prefix}-mirror"
+  owner 'root'
+  group 'root'
+  mode '0755'
+  variables({
+    :reposync_sources => reposync_sources
+  })
+end
+
+pkg = get_cron_pkg_name
+resources(:package => pkg) rescue package pkg do
+  action :install
+end
+
+template "/etc/cron.d/#{prefix}-mirror" do
+  source "etc/cron.d/#{prefix}-mirror"
+  owner 'root'
+  group 'root'
+  mode '0644'
+end
+
+httpd_service = get_httpd_pkg_name
+httpd_conf_path = nil
+
+case node[:platform_family]
+when 'debian'
+  httpd_conf_path = "/etc/#{httpd_service}/conf.d/#{prefix}-mirror"
+when 'rhel'
+  httpd_conf_path = "/etc/#{httpd_service}/conf.d/#{prefix}-mirror.conf"
+end
+
+pkg = httpd_service
+resources(:package => pkg) rescue package pkg do
+  action :install
+end
+
+resources(:service => httpd_service) rescue service httpd_service do
+  action [:enable, :start]
+  supports :status => true, :restart => true, :reload => true
+end
+
+template httpd_conf_path do
+  source "etc/apache2/conf.d/#{prefix}-mirror"
+  owner 'root'
+  group 'root'
+  mode '0644'
+  variables({
+    :url_aliases => url_aliases
+  })
+  notifies :restart, "service[#{httpd_service}]"
+end
+
diff --git a/cookbooks/yum_utils/templates/default/etc/apache2/conf.d/yum-reposync-mirror b/cookbooks/yum_utils/templates/default/etc/apache2/conf.d/yum-reposync-mirror
new file mode 100644 (file)
index 0000000..4de77bd
--- /dev/null
@@ -0,0 +1,33 @@
+#
+# Copyright 2013, whitestar
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+<%
+@url_aliases.each do |url_path, real_path|
+-%>
+Alias <%= url_path %> <%= real_path %> 
+<Directory <%= real_path %>>
+  Options Indexes SymLinksIfOwnerMatch FollowSymLinks
+  IndexOptions NameWidth=* +SuppressDescription
+  AllowOverride FileInfo Indexes
+  Order Deny,Allow
+  Allow from all
+  ErrorDocument 404 default
+</Directory>
+
+<%
+end
+-%>
+
diff --git a/cookbooks/yum_utils/templates/default/etc/cron.d/yum-reposync-mirror b/cookbooks/yum_utils/templates/default/etc/cron.d/yum-reposync-mirror
new file mode 100644 (file)
index 0000000..99d7373
--- /dev/null
@@ -0,0 +1,20 @@
+#
+# Regular cron jobs for the yum-mirror
+#
+# Copyright 2013, whitestar
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+<%= node['yum_utils']['reposync_mirror']['cron_period'] %>     <%= node['yum_utils']['reposync_mirror']['user'] %>     /usr/local/bin/yum-reposync-mirror > <%= node['yum_utils']['reposync_mirror']['base_path'] %>/var/cron.log 2>&1
+
diff --git a/cookbooks/yum_utils/templates/default/usr/local/bin/yum-reposync-mirror b/cookbooks/yum_utils/templates/default/usr/local/bin/yum-reposync-mirror
new file mode 100644 (file)
index 0000000..d342a3c
--- /dev/null
@@ -0,0 +1,29 @@
+#!/bin/sh
+#
+# Copyright 2013, whitestar
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+<%
+@reposync_sources.each do |source|
+-%>
+echo -n 'Begin time: '; date
+<%= source['reposync_command'] %>
+<%= source['createrepo_command'] %>
+echo -n 'End time: '; date
+
+<%
+end
+-%>
+
diff --git a/roles/cdh-mirror.rb b/roles/cdh-mirror.rb
new file mode 100644 (file)
index 0000000..5cc52ad
--- /dev/null
@@ -0,0 +1,29 @@
+#
+# Copyright 2013, whitestar
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name 'cdh-mirror'
+description 'CDH mirror server.'
+
+run_list(
+  'recipe[cdh::yum_mirror]',
+)
+
+#env_run_lists()
+
+default_attributes(
+)
+
+#override_attributes()
index 3c0933d..1101223 100644 (file)
@@ -25,9 +25,10 @@ run_list(
   'role[node_commons]',
   'role[apt-mirror]',
   'role[apt-cacher-ng]',
-  #'role[apt-source]',
+  'role[apt-source]',
   'role[yum-mirror]',
   #'role[yum-repos]',
+  'role[cdh-mirror]',
   'role[yum-hdp]',
   'role[hdp-mirror]',
   'role[apache-mirror]',
@@ -55,6 +56,25 @@ this_subcluster[:nodes].first(2).each {|host|
 =end
 
 default_attributes(
+  'cdh' => {
+    'version' => '4.4.0',
+    'yum_repo' => {
+      'update' => true,
+      #'only_for_mirroring' => true,
+      #'mirroring' => {
+      #  'platform' => 'centos',
+      #  'platform_version' => '6'
+      #}
+    },
+    'yum_mirror' => {
+      'repo_ids' => [
+        'cloudera-cdh4',
+        'cloudera-gplextras4',
+      ],
+      'arch' => 'x86_64',
+      'url_alias_with_authority_part' => true
+    }
+  },
   'hdp' => {
     'version' => '1.3.2',
     #'version' => '1.2.3.1',
@@ -70,8 +90,11 @@ default_attributes(
       'repo_ids' => [
         'HDP-1.x',
         'Updates-HDP-1.x',
-        'ambari-1.x'
-      ]
+        'HDP-UTILS-1.1.0.16',
+        'ambari-1.x',
+        'Updates-ambari-1.2.5.17'
+      ],
+      'url_alias_with_authority_part' => true
     }
   }
 )
@@ -85,13 +108,15 @@ override_attributes(
   'apt_utils' => {
     'source' => {
       'apt_lines' => [
-        'deb http://cdn.debian.net/debian wheezy main contrib non-free',
-        'deb http://cdn.debian.net/debian wheezy-updates main contrib',
-        'deb http://cdn.debian.net/debian wheezy-backports main contrib non-free',
+        #'deb http://cdn.debian.net/debian wheezy main contrib non-free',
+        #'deb http://cdn.debian.net/debian wheezy-updates main contrib',
+        #'deb http://cdn.debian.net/debian wheezy-backports main contrib non-free',
+        #'deb http://security.debian.org/ wheezy/updates main contrib non-free',
         # self test
-        #'deb http://localhost/debian wheezy main contrib non-free',
-        #'deb http://localhost/debian wheezy-updates main contrib',
-        #'deb http://localhost/debian wheezy-backports main contrib non-free',
+        'deb http://ftp.grid.example.com/debian wheezy main contrib non-free',
+        'deb http://ftp.grid.example.com/debian wheezy-updates main contrib',
+        'deb http://ftp.grid.example.com/debian wheezy-backports main contrib non-free',
+        'deb http://ftp.grid.example.com:3142/security.debian.org/ wheezy/updates main contrib non-free'
       ]
     },
     'mirror' => {
diff --git a/roles/yum-reposync-mirror.rb b/roles/yum-reposync-mirror.rb
new file mode 100644 (file)
index 0000000..8cb963a
--- /dev/null
@@ -0,0 +1,29 @@
+#
+# Copyright 2013, whitestar
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name 'yum-reposync-mirror'
+description 'YUM mirror server by reposync.'
+
+run_list(
+  'recipe[yum_utils::reposync_mirror]',
+)
+
+#env_run_lists()
+
+default_attributes(
+)
+
+#override_attributes()