From 7ecebdd02df9b11fa6ba4f8033dcfe097935ed66 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Mon, 12 Aug 2013 18:21:47 +0300 Subject: [PATCH] Repository import during project creation often return timeout for medium and large repos. So lets do it async. First create project, then import repo and create satellite with Sidekiq --- app/contexts/projects/create_context.rb | 14 ----- app/models/project.rb | 10 ++-- app/observers/project_observer.rb | 20 ++++--- app/views/projects/empty.html.haml | 61 +++++++++++++--------- app/workers/repository_import_worker.rb | 22 ++++++++ .../20130812143708_add_import_url_to_project.rb | 5 ++ db/schema.rb | 3 +- 7 files changed, 80 insertions(+), 55 deletions(-) create mode 100644 app/workers/repository_import_worker.rb create mode 100644 db/migrate/20130812143708_add_import_url_to_project.rb diff --git a/app/contexts/projects/create_context.rb b/app/contexts/projects/create_context.rb index aee3c1c56..64c278530 100644 --- a/app/contexts/projects/create_context.rb +++ b/app/contexts/projects/create_context.rb @@ -45,20 +45,6 @@ module Projects @project.creator = current_user - # Import project from cloneable resource - if @project.valid? && @project.import_url.present? - shell = Gitlab::Shell.new - - if shell.import_repository(@project.path_with_namespace, @project.import_url) - # We should create satellite for imported repo - @project.satellite.create unless @project.satellite.exists? - @project.imported = true - true - else - @project.errors.add(:import_url, 'cannot clone repo') - end - end - if @project.save unless @project.group @project.users_projects.create(project_access: UsersProject::MASTER, user: current_user) diff --git a/app/models/project.rb b/app/models/project.rb index 59c6d212f..5ffbead55 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -37,8 +37,6 @@ class Project < ActiveRecord::Base acts_as_taggable_on :labels, :issues_default_labels - attr_accessor :import_url - # Relations belongs_to :creator, foreign_key: "creator_id", class_name: "User" belongs_to :group, foreign_key: "namespace_id", conditions: "type = 'Group'" @@ -157,6 +155,10 @@ class Project < ActiveRecord::Base import_url.present? end + def imported? + imported + end + def check_limit unless creator.can_create_project? errors[:limit_reached] << ("Your own projects limit is #{creator.projects_limit}! Please contact administrator to increase it") @@ -411,10 +413,6 @@ class Project < ActiveRecord::Base !(forked_project_link.nil? || forked_project_link.forked_from_project.nil?) end - def imported? - imported - end - def personal? !group end diff --git a/app/observers/project_observer.rb b/app/observers/project_observer.rb index c1a461153..83c1795b1 100644 --- a/app/observers/project_observer.rb +++ b/app/observers/project_observer.rb @@ -1,13 +1,17 @@ class ProjectObserver < BaseObserver def after_create(project) - return true if project.forked? || project.imported? - - GitlabShellWorker.perform_async( - :add_repository, - project.path_with_namespace - ) - - log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"") + return true if project.forked? + + if project.import? + RepositoryImportWorker.perform_in(5.seconds, project.id) + else + GitlabShellWorker.perform_async( + :add_repository, + project.path_with_namespace + ) + + log_info("#{project.owner.name} created a new project \"#{project.name_with_namespace}\"") + end end def after_update(project) diff --git a/app/views/projects/empty.html.haml b/app/views/projects/empty.html.haml index 56dbbf075..5279ddf47 100644 --- a/app/views/projects/empty.html.haml +++ b/app/views/projects/empty.html.haml @@ -1,33 +1,42 @@ = render 'clone_panel' -%div.git-empty - %fieldset - %legend Git global setup: - %pre.dark - :preserve - git config --global user.name "#{current_user.name}" - git config --global user.email "#{current_user.email}" +- if @project.import? && !@project.imported + .save-project-loader + %center + = image_tag "ajax_loader.gif" + %h3 Importing repository. + %p.monospace git clone --bare #{@project.import_url} + %p Please wait until we import repository for you. Refresh at will. - %fieldset - %legend Create Repository - %pre.dark - :preserve - mkdir #{@project.path} - cd #{@project.path} - git init - touch README - git add README - git commit -m 'first commit' - git remote add origin #{@project.url_to_repo} - git push -u origin master +- else + %div.git-empty + %fieldset + %legend Git global setup: + %pre.dark + :preserve + git config --global user.name "#{current_user.name}" + git config --global user.email "#{current_user.email}" - %fieldset - %legend Existing Git Repo? - %pre.dark - :preserve - cd existing_git_repo - git remote add origin #{@project.url_to_repo} - git push -u origin master + %fieldset + %legend Create Repository + %pre.dark + :preserve + mkdir #{@project.path} + cd #{@project.path} + git init + touch README + git add README + git commit -m 'first commit' + git remote add origin #{@project.url_to_repo} + git push -u origin master + + %fieldset + %legend Existing Git Repo? + %pre.dark + :preserve + cd existing_git_repo + git remote add origin #{@project.url_to_repo} + git push -u origin master - if can? current_user, :remove_project, @project .prepend-top-20 diff --git a/app/workers/repository_import_worker.rb b/app/workers/repository_import_worker.rb new file mode 100644 index 000000000..a3b4bd0c9 --- /dev/null +++ b/app/workers/repository_import_worker.rb @@ -0,0 +1,22 @@ +class RepositoryImportWorker + include Sidekiq::Worker + include Gitlab::ShellAdapter + + sidekiq_options queue: :gitlab_shell + + def perform(project_id) + project = Project.find(project_id) + result = gitlab_shell.send(:import_repository, + project.path_with_namespace, + project.import_url) + + if result + project.imported = true + project.save + project.satellite.create unless project.satellite.exists? + project.discover_default_branch + else + project.imported = false + end + end +end diff --git a/db/migrate/20130812143708_add_import_url_to_project.rb b/db/migrate/20130812143708_add_import_url_to_project.rb new file mode 100644 index 000000000..023a48741 --- /dev/null +++ b/db/migrate/20130812143708_add_import_url_to_project.rb @@ -0,0 +1,5 @@ +class AddImportUrlToProject < ActiveRecord::Migration + def change + add_column :projects, :import_url, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index f6adfe3c5..e0e7d47b9 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended to check this file into your version control system. -ActiveRecord::Schema.define(:version => 20130804151314) do +ActiveRecord::Schema.define(:version => 20130812143708) do create_table "deploy_keys_projects", :force => true do |t| t.integer "deploy_key_id", :null => false @@ -178,6 +178,7 @@ ActiveRecord::Schema.define(:version => 20130804151314) do t.boolean "snippets_enabled", :default => true, :null => false t.datetime "last_activity_at" t.boolean "imported", :default => false, :null => false + t.string "import_url" end add_index "projects", ["creator_id"], :name => "index_projects_on_owner_id" -- 2.11.0