jcubic

I'm graphic designer and programmer from Poland.

Homepage: https://jcubic.wordpress.com

How backup partition on Linux with progress bar from command line

On GNU/Linux you can do everything from command line. The same goes to copy whole partition to a file. You can use powerful dd command to save your partition to a file. You can also use pv to show progress bar. Here is useful function that will backup your partition (device) to a file with progress bar (you need sudo to access device):

function backup() {
    sudo dd if=$1 | pv -s $(sudo blockdev --getsize64 $1) | bzip2 -9f > $2
}

and you can use this function like this:

backup /dev/sda1 /home/me/part.img.bz2

You can create similar function for restoring the partition:

function restore() {
    bunzip2 -dc $1 | pv -s $(ls -l $1 | cut -d' ' -f5) | dd of=$2
}

and you can use this function similar to backup (but the file is first and device second):

restore /home/me/part.img.bz2 /dev/sda1

NOTE: using dd can damage your partition, use it with caution.

,

Leave a comment

Cross-browser directory upload that can have large files

Google Chrome 21 added a feature to upload directories, Firefox also didn’t get too long far behind and added that feature in version 42. Here is a code that will work in both browsers, with help from jQuery and FormData object. It will also work with files larger then the limit:

function Uploader(path, limit) {
    this.path = path;
    this.upload_max_filesize = limit;
}

Uploader.prototype.upload_tree = function upload_tree(tree, path) {
    var defered = $.Deferred();
    var self = this;
    path = path || self.path;
    function process(entries, callback) {
        entries = entries.slice();
        (function recur() {
            var entry = entries.shift();
            if (entry) {
                callback(entry).then(recur).fail(function() {
                    defered.reject();
                });
            } else {
                defered.resolve();
            }
        })();
    }
    function upload_files(entries) {
        process(entries, function(entry) {
            return self.upload_tree(entry, path + "/" + tree.name);
        });
    }
    function upload_file(file) {
        self.upload(file, path).then(function() {
            defered.resolve();
        }).fail(function() {
            defered.reject();
        });
    }
    if (typeof Directory != 'undefined' && tree instanceof Directory) { // firefox
        tree.getFilesAndDirectories().then(function(entries) {
            upload_files(entries);
        });
    } else if (typeof File != 'undefined' && tree instanceof File) { // firefox
        upload_file(tree);
    } else if (tree.isFile) { // chrome
        tree.file(upload_file);
    } else if (tree.isDirectory) { // chrome
        var dirReader = tree.createReader();
        dirReader.readEntries(function(entries) {
            upload_files(entries);
        });
    }
    return defered.promise();
};

Uploader.prototype.upload = function upload(file, path) {
    var self = this;
    var defered = $.Deferred();
    var file_name = path + '/' + file.name;
    if (file.size > self.upload_max_filesize) {
        if (!(file.slice || file.webkitSlice)) {
            console.log('Exceeded filesize limit.');
            defered.resolve(); // next file
        } else {
            self.upload_by_chunks(file, path).then(function() {
                defered.resolve();
            }).fail(function() {
                defered.reject();
            });
        }
    } else {
        self.upload_file(file, path).then(function() {
            defered.resolve();
        }).fail(function() {
            defered.reject();
        });
    }
    return defered.promise();
};

Uploader.prototype.upload_by_chunks = function upload_by_chunks(file, path, chunk_size) {
    var self = this;
    chunk_size = chunk_size || 1048576; // 1MB
    var defered = $.Deferred();
    function slice(start, end) {
        if (file.slice) {
            return file.slice(start, end);
        } else if (file.webkitSlice) {
            return file.webkitSlice(start, end);
        }
    }
    var i = 0;
    function process(start, end) {
        if (start < file.size) {
            var chunk = slice(start, end);
            var formData = new FormData();
            formData.append('file', chunk, file.name);
            formData.append('token', self.token);
            formData.append('path', path);
            $.ajax({
                url: 'lib/upload.php?append=1',
                type: 'POST',
                success: function(response) {
                    if (response.error) {
                        console.log(response.error);
                        defered.reject();
                    } else {
                        process(end, end+chunk_size);
                    }
                },
                error: function(jxhr, error, status) {
                    console.log(jxhr.statusText);
                    defered.reject();
                },
                data: formData,
                cache: false,
                contentType: false,
                processData: false
            });
        } else {
            console.log('File "' + file.name + '" uploaded.');
            defered.resolve();
        }
    }
    var fname = path + '/' + file.name;
    // we need to remove the file if already there
    $.ajax({
        url: 'delete.php',
        data: {file: fname},
        success: function(response) {
            if (response.result) {
                process(0, chunk_size);
            } else if (response.error) {
                console.log(response.error);
                defered.reject();
            }
        }
    });
    return defered.promise();
};

Uploader.prototype.upload_file = function upload_file(file, path) {
    var self = this;
    var defered = $.Deferred();
    var formData = new FormData();
    formData.append('file', file);
    formData.append('path', path);
    $.ajax({
        url: 'lib/upload.php',
        type: 'POST',
        success: function(response) {
            if (response.error) {
                console.log(response.error);
                defered.reject();
            } else {
                console.log('File "' + file.name + '" ' + 'uploaded.');
                defered.resolve();
            }
        },
        error: function(jxhr, error, status) {
            console.log(jxhr.statusText);
            defered.reject();
        },
        data: formData,
        cache: false,
        contentType: false,
        processData: false
    });
    return defered.promise();
};

Here is drag and drop code that use that “class”:

$('div').on('drop', function(e) {
    e.preventDefault();
    var org = e.originalEvent;
    if (!terminal.token()) {
        return;
    }
    var uploader = new Uploader('/home/user', FILE_LIMIT_TAKEN_FROM_SERVER);
    var items;
    if (org.dataTransfer.items) {
        items = [].slice.call(org.dataTransfer.items);
    }
    var files = (org.dataTransfer.files || org.target.files);
    if (files) {
        files = [].slice.call(files);
    }
    if (items && items.length) {
        if (items[0].webkitGetAsEntry) {
            (function upload() {
                var item = items.shift();
                if (item) {
                    item = item.webkitGetAsEntry();
                    uploader.upload_tree(item).then(upload)
                } else {
                    console.log('Finish');
                }
            })();
        }
    } else if (files && files.length) {
        (function upload() {
            var file = files.shift();
            if (file) {
                uploader.upload(file).then(upload)
            } else {
                console.log('Finish');
            }
        });
    } else if (org.dataTransfer.getFilesAndDirectories) {
        org.dataTransfer.getFilesAndDirectories().then(function(items) {
            (function upload() {
                var item = items.shift();
                if (item) {
                    uploader.upload_tree(item).then(upload);
                } else {
                    console.log('Finish');
                }
            })();
        });
    }
}).on('dragover', function(e) {
    e.preventDefault();
}).on('dragenter', function(e) {
    e.preventDefault();
});

upload.php can look like this:

<?php
header('Content-type: application/json');

if (!isset($_FILES['file'])) {
    echo json_encode(array('error' => 'No File'));
} else if (!isset($_POST['path'])) {
    echo json_encode(array('error' => 'Wrong request, no path'));
} else {
    $fname = basename($_FILES['file']['name']);
    switch ($_FILES['file']['error']) {
        case UPLOAD_ERR_OK:
            $path = '';
            // create directories if don't exists
            foreach (explode("/", $_POST['path']) as $folder) {
                if (!is_dir($path . DIRECTORY_SEPARATOR . $folder)) {
                    mkdir($path . DIRECTORY_SEPARATOR . $folder);
                }
                $path .= DIRECTORY_SEPARATOR . $folder;
            }
            $full_name = $_POST['path'] . '/' . $fname;
            if (file_exists($full_name) && !is_writable($full_name)) {
                echo json_encode(array(
                    'error' => 'File "'. $fname . '" is not writable'
                ));
            } else {
                if (isset($_GET['append'])) {
                    $contents = file_get_contents($_FILES['file']['tmp_name']);
                    $file = fopen($full_name, 'a+');
                    if (!$file) {
                        echo json_encode(array('error' => 'Can\'t save file.'));
                    } else if (fwrite($file, $contents) != strlen($contents)) {
                        echo json_encode(array('error' => 'Not all bytes saved.'));
                    } else {
                        echo json_encode(array('success' => true));
                    }
                    fclose($file);
                } else {
                    if (!move_uploaded_file($_FILES['file']['tmp_name'],
                                            $full_name)) {
                        echo json_encode(array('error' => 'Can\'t save file.'));
                    } else {
                        echo json_encode(array('success' => true));
                    }
                }
            }
            break;
        case UPLOAD_ERR_NO_FILE:
            echo json_encode(array('error' => 'File not sent.'));
            break;
        case UPLOAD_ERR_INI_SIZE:
        case UPLOAD_ERR_FORM_SIZE:
            echo json_encode(array('error' => 'Exceeded filesize limit.'));
            break;
        default:
            echo json_encode(array('error' => 'Unknown error.'));
    }
}
?>

The code for delete can just call unlink function if file exist.

Another thing that can be added to the code, is to ask a question if overwrite a file if it exists. This is left as a exercise to the reader.

, ,

Leave a comment

Debugging code that call $resource in Angular with a Proxy

In angular you can add REST service using $resource factory, for instance you can create new REST object like this:

var studies =  $resource('http://example.com/api/studies/:id', {
  id: '@id'
}, {
  get: {
    method: 'GET',
    cache: false,
    interceptor: {
      response: doInterceptStudy
    }
  },
  query: {
    method: 'GET',
    cache: false,
    params: {
      size: 20
    }
  },
  lasarLookup: {
    bypassUiBlockInterceptor: true,
    method: 'GET',
    cache: false,
    params: {
      size: 20
    }
  },
  filteredQueryAll: {
    url: 'http://example.com/api/studies',
    method: 'POST',
    cache: false,
    params: {
      size: 20
    }
  }
});

and then call it using:

studies.get({id:123}, function(user) {
  $scope.user = user;
});

In chrome you can add breakpoint on XHR request but you will probably end up in angular code when you do that.

It’s probably better ot add breakpoint to lines where you call your $resource like studies.get, but if you have responses with lot of endpoints that are called in different places this is troublesome. But there is quick solution to add breakpoint on every call to resource with a ES6 proxy

studies = new Proxy(studies, {
    get: function(target, name) {
        if (typeof target[name] == 'function') {
            return function() {
                return target[name].apply(target, [].slice.call(arguments));
            };
        } else {
            return target[name];
        }
    }
 });

and you can add breakpoint inside function returned by get proxy method or you can add dubbugger statement:

studies = new Proxy(studies, {
    get: function(target, name) {
        if (typeof target[name] == 'function') {
            return function() {
                debugger;
                return target[name].apply(target, [].slice.call(arguments));
            };
        } else {
            return target[name];
        }
    }
 });

The support for Proxies is pretty decent, only IE don’t support it. You can check browser support on can I use.

,

Leave a comment

How to replace tabs by spaces in whole codebase?

I’ve stared working on a project on github, written in php, that have mixed spaces and tabs for indent, I’m using GNU/Linux so I thought that I just replace all tabs by spaces using bash. Here is the command I’ve used that just did that:

find . -type f -name '*.php' | while read file; do
     grep $'\t' $file > /dev/null && sed -ie 's/\t/    /g' $file
done

Explanation:

  • find . -type f -name '*.php' – this will search for all files with php extension, in current directory. I’ve used -type f to return only files since there where directories that end with php.
  • while read file; do ... done – while loop over files found by find, the filename will be in file variable
  • grep $'\t' $file > /dev/null – this will return true if file contain tabs
  • & & execute next command if previous is true
  • sed -ie 's/\t/ /g' $file – replace tabs by spaces in file inline (-i option)

, , , ,

Leave a comment

Get list of github contributors in node.js

Recently I needed to send emails to all contributors to my github project, so I wrote a little script in node.js (actually my first try was ruby, but have problems with https). Here is the script using only native modules:

#!/usr/bin/node

var https = require('https');
var path = require('path');
function get(host, path, callback) {
    var options = {
        host: host,
        path: path,
        headers: {
            'User-Agent': 'Node.js' //required by github api
        }
    }
    https.get(options, function(res) {
        var output = '';
        res.setEncoding('utf8');

        res.on('data', function (chunk) {
            output += chunk;
        });

        res.on('end', function() {
            callback(JSON.parse(output));
        });
    });
}

if (process.argv.length == 4) {
    var user = process.argv[2];
    var repo = process.argv[3];
    var path = '/repos/' + user + '/' + repo + '/contributors'
    get('api.github.com', path, function(contributors) {
        contributors.forEach(function(contributor) {
            if (contributor.login != user) {
                var path = contributor.url.replace(/https:\/\/[^\/]+/, '');
                get('api.github.com', path, function(user) {
                    if (user.name) {
                        var email = user.email ? ' <' + user.email + '>' : ''; 
                        console.log(user.name + email);
                    }
                });
            }
        });
    });
} else {
    console.log('usage: \n' + path.basename(process.argv[1]) + ' user repo');
}

, ,

Leave a comment

Reading GET, POST and Cookie values in guile scheme

I’ve start playing with cgi in guile scheme and I came up with the functions to read GET, POST and Cookie values that I want to share. here is the code:

(use-modules (web uri)) ;; includes uri-decode procedure

(define (read-port port)
    (let iter ((result '()) (chr (read-char port)))
        (if (eof-object? chr)
            (list->string result)
            (iter (append result (list chr)) (read-char port)))))

(define (params->alist string fn sep)
  (map (lambda (pair)
          (let* ((list (string-split pair #\=))
                 (key (fn (string-trim (car list))))
                 (value (fn (string-trim (string-join (cdr list) "=")))))
             (cons key value)))
       (string-split string sep)))

(define (create-param-proc string fn sep)
   (if (or (equal? string #f) (equal? string ""))
       (lambda (var) "")
       (let ((query (params->alist string fn sep)))
          (lambda (var)
             (let ((pair (assoc var query)))
               (if (null? pair)
                   ""
                   (cdr pair)))))))



(define GET (create-param-proc (getenv "QUERY_STRING") uri-decode #\&))
(define POST (create-param-proc (read-port (current-input-port)) uri-decode #\&))
(define COOKIE (create-param-proc (getenv "HTTP_COOKIE") identity #\;))

And to get the values you just execute one of GET, POST, COOKIE procedures:

(display (string-append "foo: " (GET "foo")))
(newline)

(display (string-append "foo: " (POST "foo")))
(newline)
(display (string-append "foo: " (COOKIE "foo")))
(newline)

, , , ,

Leave a comment

Async shell command execution in GNU Emacs

In GNU Emacs if you execute shell command for instance using exec function the whole Editor freeze until shell command is finished. So I’ve written asynchronous version of exec:

(defun async-exec-command (command &rest success)
  (interactive "MExecute command: ")
  (let* ((buffer-name (generate-new-buffer-name "**shell**"))
         (buffer (get-buffer-create buffer-name))
         (process (apply #'start-process
                         (append (list buffer-name buffer)
                                 (split-string command " ")))))
    (lexical-let ((buffer buffer) (success (car success)) (command command))
      (set-process-sentinel process
                            (if success
                                (lambda (process str)
                                  (save-excursion
                                    (set-buffer buffer)
                                    (let ((content (buffer-string)))
                                      (kill-buffer buffer)
                                      (if (or (string= str "finished\n")
                                              (string-match "exited abnormally" str))
                                          (funcall success content)
                                        (message content)))))
                              (lambda (proces str)
                                (kill-buffer buffer)))))
    (concat "execute: " command)))

, ,

Leave a comment

%d bloggers like this: