added amazon s3 module

2010-11-26 22:44:32 +00:00 · 2010-11-26 22:44:32 +00:00 · df26355f00
commit df26355f00
parent 34474a24ee
12 changed files with 1916 additions and 0 deletions
--- a/3.0/modules/aws_s3/controllers/admin_aws_s3.php
+++ b/3.0/modules/aws_s3/controllers/admin_aws_s3.php
@ -0,0 +1,124 @@
+<?php
+
+class Admin_Aws_S3_Controller extends Admin_Controller {
+
+    public function index() {
+        // require_once(MODPATH . "aws_s3/lib/s3.php");
+        
+        $form = $this->_get_s3_form();
+
+        if (request::method() == "post") {
+            access::verify_csrf();
+
+            if ($form->validate()) {
+                module::set_var("aws_s3", "enabled",     (isset($_POST['enabled']) ? true : false));
+                module::set_var("aws_s3", "access_key",  $_POST['access_key']);
+                module::set_var("aws_s3", "secret_key",  $_POST['secret_key']);
+                module::set_var("aws_s3", "bucket_name", $_POST['bucket_name']);
+                module::set_var("aws_s3", "g3id",        $_POST['g3id']);
+
+                module::set_var("aws_s3", "url_str",     $_POST['url_str']);
+                module::set_var("aws_s3", "sig_exp",     $_POST['sig_exp']);
+
+                module::set_var("aws_s3", "use_ssl",     (isset($_POST['use_ssl']) ? true : false));
+
+                if (module::get_var("aws_s3", "enabled") && !module::get_var("aws_s3", "synced", false))
+                        site_status::warning(
+                          t('Your site has not yet been syncronised with your Amazon S3 bucket. Content will not appear correctly until you perform syncronisation. <a href="%url" class="g-dialog-link">Fix this now</a>',
+                            array("url" => html::mark_clean(url::site("admin/maintenance/start/aws_s3_task::sync?csrf=__CSRF__")))
+                          ), "aws_s3_not_synced");
+
+
+                message::success(t("Settings have been saved"));
+                url::redirect("admin/aws_s3");
+            }
+            else {
+                message::error(t("There was a problem with the submitted form. Please check your values and try again."));
+            }
+        }
+        
+        $v = new Admin_View("admin.html");
+        $v->page_title = t("Amazon S3 Configuration");
+        $v->content = new View("admin_aws_s3.html");
+        $v->content->form = $form;
+        $v->content->end = "";
+
+        echo $v;
+    }
+
+    private function _get_s3_form() {
+        $form = new Forge("admin/aws_s3", "", "post", array("id" => "g-admin-s3-form"));
+
+        $group = $form->group("aws_s3")->label(t("Amazon S3 Settings"));
+
+        $group  ->checkbox("enabled")
+                ->id("s3-enabled")
+                ->checked(module::get_var("aws_s3", "enabled"))
+                ->label("S3 enabled");
+
+        $group  ->input("access_key")
+                ->id("s3-access-key")
+                ->label("Access Key ID")
+                ->value(module::get_var("aws_s3", "access_key"))
+                ->rules("required")
+                ->error_messages("required", "This field is required")
+                ->message('<a target="_blank" href="https://aws-portal.amazon.com/gp/aws/developer/account/index.html?ie=UTF8&action=access-key">Sign up to Amazon S3</a>');
+
+        $group  ->input("secret_key")
+                ->id("s3-secret-key")
+                ->label("Secret Access Key")
+                ->value(module::get_var("aws_s3", "secret_key"))
+                ->rules("required")
+                ->error_messages("required", "This field is required");
+
+        $group  ->input("bucket_name")
+                ->id("s3-bucket")
+                ->label("Bucket Name")
+                ->value(module::get_var("aws_s3", "bucket_name"))
+                ->rules("required")
+                ->error_messages("required", "This field is required")
+                ->message("Note: This module will not create a bucket if it does not already exist. Please ensure you have already created the bucket and the bucket has the correct ACL permissions before continuing.");
+
+        $group  ->input("g3id")
+                ->id("s3-g3id")
+                ->label("G3 ID")
+                ->value(module::get_var("aws_s3", "g3id", md5(time())))
+                ->rules("required")
+                ->error_messages("required", "This field is required")
+                ->message("This field allows for multiple G3 instances running off of a single S3 bucket.");
+
+        $group  ->checkbox("use_ssl")
+                ->id("s3-use-ssl")
+                ->checked(module::get_var("aws_s3", "use_ssl"))
+                ->label("Use SSL for S3 transfers");
+
+        $group = $form->group("cdn_settings")->label(t("CDN Settings"));
+
+        $group  ->input("url_str")
+                ->id("s3-url-str")
+                ->label("URL String")
+                ->value(module::get_var("aws_s3", "url_str", "http://{bucket}.s3.amazonaws.com/g3/{guid}/{resource}"))
+                ->rules("required")
+                ->message("Configure the URL to access uploaded resources on the CDN. Use the following variables to define and build up the URL:<br />
+&bull; {bucket} - Bucket Name<br />
+&bull; {guid} - Unique identifier for this gallery installation<br />
+&bull; {resource} - The end path to the resource/object");
+
+        $group  ->input("sig_exp")
+                ->id("sig_exp")
+                ->label("Private Content Signature Duration")
+                ->value(module::get_var("aws_s3", "sig_exp", 60))
+                ->rules("required")
+                ->callback("aws_s3::validate_number")
+                ->error_messages("not_numeric", "The value provided is not numeric. Please enter a number in this field.")
+                ->message("Set the time in seconds for the generated signature for access to permission-restricted S3 objects<br /><br />
+Note: this module does not yet support the creation of signatures to access private objects on S3 via CloudFront CDN.");
+
+        $form   ->submit("save")
+                ->value("Save Settings");
+        
+
+        return $form;
+    }
+
+}
--- a/3.0/modules/aws_s3/helpers/MY_embedlinks_block.php
+++ b/3.0/modules/aws_s3/helpers/MY_embedlinks_block.php
@ -0,0 +1,10 @@
+<?php
+
+class embedlinks_block extends embedlinks_block_Core {
+
+    static function get($block_id, $theme) {
+        if ($theme->item && $theme->item->view_1 == 1)
+            parent::get($block_id, $theme);
+    }
+
+}
--- a/3.0/modules/aws_s3/helpers/MY_embedlinks_theme.php
+++ b/3.0/modules/aws_s3/helpers/MY_embedlinks_theme.php
@ -0,0 +1,13 @@
+<?php
+
+class embedlinks_theme extends embedlinks_theme_Core {
+
+    static function photo_bottom($theme) {
+        if (module::get_var("embedlinks", "InPageLinks")) {
+            $item = $theme->item;
+            if ($item->view_1 == 1)
+                return parent::photo_bottom($theme);
+        }
+    }
+
+}
--- a/3.0/modules/aws_s3/helpers/MY_item.php
+++ b/3.0/modules/aws_s3/helpers/MY_item.php
@ -0,0 +1,23 @@
+<?php
+
+class item extends item_Core {
+
+    static function make_album_cover($item, $sync = false) {
+        if (!$sync)
+            parent::make_album_cover($item);
+
+        $parent = $item->parent();
+        if ($parent->id > 1) {
+            aws_s3::upload_album_cover($parent);
+        }
+    }
+
+    static function remove_album_cover($album) {
+        parent::remove_album_cover($album);
+
+        if ($album->id > 1) {
+            aws_s3::remove_album_cover($album);
+        }
+    }
+
+}
--- a/3.0/modules/aws_s3/helpers/aws_s3.php
+++ b/3.0/modules/aws_s3/helpers/aws_s3.php
@ -0,0 +1,173 @@
+<?php
+
+class aws_s3_Core {
+
+    private static $_s3;
+
+    static function get_s3() {
+        if (!self::$_s3) {
+            require_once(MODPATH . "aws_s3/lib/s3.php");
+            S3::setAuth(module::get_var("aws_s3", "access_key"), module::get_var("aws_s3", "secret_key"));
+            S3::$useSSL = module::get_var("aws_s3", "use_ssl", false);
+        }
+        return self::$_s3;
+    }
+
+    static function getHash($string) {
+        return base64_encode(extension_loaded('hash') ?
+            hash_hmac('sha1', $string, module::get_var("aws_s3", "secret_key"), true) : pack('H*', sha1(
+            (str_pad(module::get_var("aws_s3", "secret_key"), 64, chr(0x00)) ^ (str_repeat(chr(0x5c), 64))) .
+            pack('H*', sha1((str_pad(module::get_var("aws_s3", "secret_key"), 64, chr(0x00)) ^
+            (str_repeat(chr(0x36), 64))) . $string)))));
+    }
+    
+    static function generate_url($resource, $authenticated = false, $updated = null) {
+        $find = array("{guid}", "{bucket}", "{resource}");
+        $replace = array(module::get_var("aws_s3", "g3id"), module::get_var("aws_s3", "bucket_name"), $resource);
+        $url = str_replace($find, $replace, module::get_var("aws_s3", "url_str"));
+        
+        if ($authenticated) {
+            preg_match("%https?://([a-zA-Z0-9\.-]*)/(.*)$%", $url, $matches);
+            $host = module::get_var("aws_s3" , "bucket_name");
+            $resource = $matches[2];
+            $url .= "?AWSAccessKeyId=" . module::get_var("aws_s3", "access_key") .
+                    "&Expires=" . (time() + module::get_var("aws_s3", "sig_exp")) .
+                    "&Signature=" . urlencode(self::getHash("GET\n\n\n" . (time() + module::get_var("aws_s3", "sig_exp")) . "\n/" . $host . "/" . $resource));
+            
+            self::get_s3();
+            S3::getAuthenticatedURL("danneh-org", $resource, module::get_var("aws_s3", "sig_exp"));
+        }
+        else
+            $url .= "?m=" . ($updated ? $updated : time());
+
+        return $url;
+    }
+
+    static function get_resource_url($resource) {
+        $url = self::generate_url($resource);
+        preg_match("%https?://[\w\.\-]*/(.*)\?%", $url, $matches);
+        if (count($matches) > 0)
+            return $matches[1];
+        return false;
+    }
+
+    static function log($item) {
+        if (is_string($item) || is_numeric($item)) {}
+        else
+            $item = print_r($item, true);
+
+        $fh = fopen(VARPATH . "modules/aws_s3/log/aws_s3-" . date("Y-m-d") . ".log", "a");
+        fwrite($fh, date("Y-m-d H:i:s") . ": " . $item .  "\n");
+        fclose($fh);
+    }
+
+    static function upload_item($item) {
+        self::get_s3();
+
+        $success_fs = S3::putObjectFile(VARPATH . "albums/" . $item->relative_path(),
+                                       module::get_var("aws_s3", "bucket_name"),
+                                       self::get_resource_url("fs/" . $item->relative_path()),
+                                       ($item->view_1 ? S3::ACL_PUBLIC_READ : S3::ACL_PRIVATE));
+        $success_th = S3::putObjectFile(VARPATH . "thumbs/" . $item->relative_path(),
+                                       module::get_var("aws_s3", "bucket_name"),
+                                       self::get_resource_url("th/" . $item->relative_path()),
+                                       ($item->view_1 ? S3::ACL_PUBLIC_READ : S3::ACL_PRIVATE));
+        $success_rs = S3::putObjectFile(VARPATH . "resizes/" . $item->relative_path(),
+                                       module::get_var("aws_s3", "bucket_name"),
+                                       self::get_resource_url("rs/" . $item->relative_path()),
+                                       ($item->view_1 ? S3::ACL_PUBLIC_READ : S3::ACL_PRIVATE));
+
+        $success = $success_fs && $success_th && $success_rs;
+        aws_s3::log("item upload success: " . $success);
+    }
+
+    static function move_item($old_item, $new_item) {
+        self::get_s3();
+
+        S3::copyObject(module::get_var("aws_s3", "bucket_name"),
+                       self::get_resource_url("fs/" . $old_item->relative_path()),
+                       module::get_var("aws_s3", "bucket_name"),
+                       self::get_resource_url("fs/" . $new_item->relative_path()),
+                       ($new_item->view_1 ? S3::ACL_PUBLIC_READ : S3::ACL_PRIVATE));
+        S3::deleteObject(module::get_var("aws_s3", "bucket_name"),
+                         self::get_resource_url("fs/" . $old_item->relative_path()));
+
+        S3::copyObject(module::get_var("aws_s3", "bucket_name"),
+                       self::get_resource_url("rs/" . $old_item->relative_path()),
+                       module::get_var("aws_s3", "bucket_name"),
+                       self::get_resource_url("rs/" . $new_item->relative_path()),
+                       ($new_item->view_1 ? S3::ACL_PUBLIC_READ : S3::ACL_PRIVATE));
+        S3::deleteObject(module::get_var("aws_s3", "bucket_name"),
+                         self::get_resource_url("rs/" . $old_item->relative_path()));
+
+        S3::copyObject(module::get_var("aws_s3", "bucket_name"),
+                       self::get_resource_url("th/" . $old_item->relative_path()),
+                       module::get_var("aws_s3", "bucket_name"),
+                       self::get_resource_url("th/" . $new_item->relative_path()),
+                       ($new_item->view_1 ? S3::ACL_PUBLIC_READ : S3::ACL_PRIVATE));
+        S3::deleteObject(module::get_var("aws_s3", "bucket_name"),
+                         self::get_resource_url("th/" . $old_item->relative_path()));
+    }
+
+    static function remove_item($item) {
+        self::get_s3();
+
+        $success_fs = S3::deleteObject(module::get_var("aws_s3", "bucket_name"),
+                                       self::get_resource_url("fs/" . $item->relative_path()));
+        $success_th = S3::deleteObject(module::get_var("aws_s3", "bucket_name"),
+                                       self::get_resource_url("th/" . $item->relative_path()));
+        $success_rs = S3::deleteObject(module::get_var("aws_s3", "bucket_name"),
+                                       self::get_resource_url("rs/" . $item->relative_path()));
+
+        $success = $success_fs && $success_th && $success_rs;
+        aws_s3::log("s3 delete success: " . $success);
+    }
+
+    static function upload_album_cover($album) {
+        self::get_s3();
+
+        if (file_exists(VARPATH . "resizes/" . $album->relative_path() . "/.album.jpg"))
+            $success_rs = S3::putObjectFile(VARPATH . "resizes/" . $album->relative_path() . "/.album.jpg",
+                                           module::get_var("aws_s3", "bucket_name"),
+                                           "g3/" . module::get_var("aws_s3", "g3id") . "/rs/" . $album->relative_path() . "/.album.jpg",
+                                           ($album->view_1 ? S3::ACL_PUBLIC_READ : S3::ACL_PRIVATE));
+        else
+            $success_rs = true;
+
+        if (file_exists(VARPATH . "thumbs/" . $album->relative_path() . "/.album.jpg"))
+            $success_th = S3::putObjectFile(VARPATH . "thumbs/" . $album->relative_path() . "/.album.jpg",
+                                           module::get_var("aws_s3", "bucket_name"),
+                                           "g3/" . module::get_var("aws_s3", "g3id") . "/th/" . $album->relative_path() . "/.album.jpg",
+                                           ($album->view_1 ? S3::ACL_PUBLIC_READ : S3::ACL_PRIVATE));
+        else
+            $success_th = true;
+
+        $success = $success_rs && $success_th;
+        aws_s3::log("album cover upload success: " . $success);
+    }
+
+    static function remove_album_cover($album) {
+        self::get_s3();
+
+        $success_th = S3::deleteObject(module::get_var("aws_s3", "bucket_name"),
+                                      "g3/" . module::get_var("aws_s3", "g3id") . "/th/" . $album->relative_path() . "/.album.jpg");
+        $success_rs = S3::deleteObject(module::get_var("aws_s3", "bucket_name"),
+                                      "g3/" . module::get_var("aws_s3", "g3id") . "/rs/" . $album->relative_path() . "/.album.jpg");
+
+        $success = $success_rs && $success_th;
+        aws_s3::log("album cover removal success: " . $success);
+    }
+
+    static function getAuthenticatedURL($bucket, $uri) {
+        self::get_s3();
+
+        return S3::getAuthenticatedURL($bucket, $uri, 60);
+    }
+
+    static function validate_number($field) {
+        if (preg_match("/\D/", $field->value))
+                $field->add_error("not_numeric", 1);
+    }
+
+
+}
--- a/3.0/modules/aws_s3/helpers/aws_s3_event.php
+++ b/3.0/modules/aws_s3/helpers/aws_s3_event.php
@ -0,0 +1,34 @@
+<?php
+
+class aws_s3_event_Core {
+
+    static function admin_menu($menu, $theme) {
+        $menu
+            ->get("settings_menu")
+            ->append(
+                    Menu::factory("link")
+                            ->id("aws_s3_link")
+                            ->label(t("Amazon S3"))
+                            ->url(url::site("admin/aws_s3"))
+                    );
+    }
+
+    static function item_created($item) {
+        if ($item->is_album())
+            return true;
+
+        aws_s3::log("Item created - " . $item->id);
+        aws_s3::upload_item($item);
+    }
+
+    static function item_deleted($item) {
+        aws_s3::log("Item deleted - " . $item->id);
+        aws_s3::remove_item($item);
+    }
+
+    static function item_moved($new_item, $old_item) {
+        aws_s3::log("Item moved - " . $item->id);
+        aws_s3::move_item($old_item, $new_item);
+    }
+
+}
--- a/3.0/modules/aws_s3/helpers/aws_s3_installer.php
+++ b/3.0/modules/aws_s3/helpers/aws_s3_installer.php
@ -0,0 +1,40 @@
+<?php
+
+class aws_s3_installer {
+
+    private static function getversion() { return 1; }
+    private static function setversion() { module::set_version("aws_s3", self::getversion()); }
+
+    static function install() {
+        @mkdir(VARPATH . "modules/aws_s3");
+        @mkdir(VARPATH . "modules/aws_s3/log");
+
+        // installation's unique identifier - allows multiple g3's pointing to the same s3 bucket.
+        if (!module::get_var("aws_s3", "g3id"))
+            module::set_var("aws_s3", "g3id", md5(time()));
+
+        module::set_var("aws_s3", "synced", false);
+        module::set_var("aws_s3", "enabled", false);
+        module::set_var("aws_s3", "access_key", "");
+        module::set_var("aws_s3", "secret_key", "");
+        module::set_var("aws_s3", "bucket_name", "");
+
+        self::setversion();
+    }
+    static function uninstall() {
+        dir::unlink(VARPATH . "modules/aws_s3");
+    }
+
+    static function upgrade($version) {
+        if ($version < self::getversion())
+            self::setversion();
+    }
+
+    static function deactivate() {}
+    static function activate() {}
+    static function can_activate() {
+        $messages = array();
+        return $messages;
+    }
+
+}
--- a/3.0/modules/aws_s3/helpers/aws_s3_task.php
+++ b/3.0/modules/aws_s3/helpers/aws_s3_task.php
@ -0,0 +1,78 @@
+<?php
+
+class aws_s3_task_Core {
+
+    static function available_tasks() {
+          return array(Task_Definition::factory()
+                   ->callback("aws_s3_task::sync")
+                   ->name(t("Syncronise with Amazon S3"))
+                   ->description(t("Syncronise your Gallery 3 data/images with your Amazon S3 bucket"))
+                   ->severity(log::SUCCESS));
+    }
+
+    static function sync($task) {
+        require_once(MODPATH . "aws_s3/lib/s3.php");
+        $s3 = new S3(module::get_var("aws_s3", "access_key"), module::get_var("aws_s3", "secret_key"));
+        
+        $mode = $task->get("mode", "init");
+        switch ($mode) {
+            case "init": {
+                aws_s3::log("re-sync task started..");
+                batch::start();
+                $items = ORM::factory("item")->find_all();
+                aws_s3::log("items to sync: " . count($items));
+                $task->set("total_count", count($items));
+                $task->set("completed", 0);
+                $task->set("mode", "empty");
+                $task->status = "Emptying contents of bucket";
+            } break;
+            case "empty": { // 0 - 10%
+                aws_s3::log("emptying bucket contents (any files that may already exist in the bucket/prefix path)");
+                $bucket = module::get_var("aws_s3", "bucket_name");
+
+                $resource = aws_s3::get_resource_url("");
+                $stuff = array_reverse(S3::getBucket($bucket, $resource));
+                foreach ($stuff as $uri => $item) {
+                    aws_s3::log("removing: " . $uri);
+                    S3::deleteObject($bucket, $uri);
+                }
+                $task->percent_complete = 10;
+                $task->set("mode", "upload");
+                $task->state = "Commencing upload...";
+            } break;
+            case "upload": { // 10 - 100%
+                $completed = $task->get("completed", 0);
+                $items = ORM::factory("item")->find_all(1, $completed);
+                foreach ($items as $item) {
+                    if ($item->id > 1) {
+                        aws_s3::log("uploading item " . $item->id . " (" . ($completed + 1)  . "/" . $task->get("total_count") . ")");
+                        if ($item->is_album())
+                            aws_s3::upload_album_cover($item);
+                        else
+                            aws_s3::upload_item($item);
+                    }
+                    $completed++;
+                }
+                $task->set("completed", $completed);
+                $task->percent_complete = round(90 * ($completed / $task->get("total_count"))) + 10;
+                $task->status = $completed . " of " . $task->get("total_count"). " uploaded.";
+
+                if ($completed == $task->get("total_count")) {
+                    $task->set("mode", "finish");
+                }
+            } break;
+            case "finish": {
+                aws_s3::log("completing upload task..");
+                $task->percent_complete = 100;
+                $task->state = "success";
+                $task->done = true;
+                $task->status = "Sync task completed successfully";
+                batch::stop();
+                module::set_var("aws_s3", "synced", true);
+                site_status::clear("aws_s3_not_synced");
+            } break;
+        }
+        
+    }
+
+}
--- a/3.0/modules/aws_s3/lib/s3.php
+++ b/3.0/modules/aws_s3/lib/s3.php
--- a/3.0/modules/aws_s3/models/MY_Item_Model.php
+++ b/3.0/modules/aws_s3/models/MY_Item_Model.php
@ -0,0 +1,40 @@
+<?php
+
+class Item_Model extends Item_Model_Core {
+
+    public function thumb_url($full_uri=false) {
+        if (!module::get_var("aws_s3", "enabled"))
+            return parent::thumb_url($full_uri);
+
+        if ($this->is_photo()) {
+            return aws_s3::generate_url("th/" . $this->relative_path(), ($this->view_1 == 1 ? false : true), $this->updated);
+        }
+        else if ($this->is_album() && $this->id > 1) {
+            return aws_s3::generate_url("th/" . $this->relative_path() . "/.album.jpg", ($this->view_1 == 1 ? false : true), $this->updated);
+        }
+        else if ($this->is_movie()) {
+            $relative_path = preg_replace("/...$/", "jpg", $this->relative_path());
+            return aws_s3::generate_url("th/" . $relative_path, ($this->view_1 == 1 ? false : true), $this->updated);
+        }
+    }
+
+    public function file_url($full_uri=false) {
+        if (!module::get_var("aws_s3", "enabled"))
+            return parent::file_url($full_uri);
+
+        return aws_s3::generate_url("fs/" . $this->relative_path(), ($this->view_1 == 1 ? false : true), $this->updated);
+    }
+
+    public function resize_url($full_uri=false) {
+        if (!module::get_var("aws_s3", "enabled"))
+            return parent::resize_url($full_uri);
+
+        if ($this->is_album() && $this->id > 1) {
+            return aws_s3::generate_url("rs/" . $this->relative_path() . "/.album.jpg", ($this->view_1 == 1 ? false : true), $this->updated);
+        }
+        else {
+            return aws_s3::generate_url("rs/" . $this->relative_path(), ($this->view_1 == 1 ? false : true), $this->updated);
+        }
+    }
+  
+}
--- a/3.0/modules/aws_s3/module.info
+++ b/3.0/modules/aws_s3/module.info
@ -0,0 +1,3 @@
+name = "Amazon S3"
+description = "Seamlessly transfer your Gallery data to Amazon S3 CDN for a lightning fast gallery"
+version = 1
--- a/3.0/modules/aws_s3/views/admin_aws_s3.html.php
+++ b/3.0/modules/aws_s3/views/admin_aws_s3.html.php
@ -0,0 +1,13 @@
+<?php defined("SYSPATH") or die("No direct script access.") ?>
+
+<div id="g-admin-code-block">
+    <h2><?= t("Amazon S3") ?></h2>
+
+    <p><?php echo t("Amazon S3 is a lightning fast Content Delivery Network. It's used for high-traffic sites to offload the bandwidth and processing required to vend vast quantities of data (pictures, videos, etc) to the cloud, leaving the local server only the tasks of running Gallery and vending small HTML pages."); ?></p>
+
+    <div class="g-block-content">
+        <?php echo $form; ?>
+    </div>
+</div>
+
+<?php echo $end; ?>