ext4_utils: Add support for >2G input files

Change-Id: I6af69d676610099d3912e90fcab3cbdc27ace4e2

diff --git a/ext4_utils/allocate.c b/ext4_utils/allocate.c
index 5f3f12b..9377e3c 100644 (file)
--- a/ext4_utils/allocate.c
+++ b/ext4_utils/allocate.c
@@ -14,14 +14,14 @@
  * limitations under the License.
  */
 
-#include <stdio.h>
-#include <stdlib.h>
-
 #include "ext4_utils.h"
 #include "allocate.h"
 #include "backed_block.h"
 #include "ext4.h"
 
+#include <stdio.h>
+#include <stdlib.h>
+
 struct region_list {
        struct region *first;
        struct region *last;

diff --git a/ext4_utils/allocate.h b/ext4_utils/allocate.h
index f4bed41..16dc33c 100644 (file)
--- a/ext4_utils/allocate.h
+++ b/ext4_utils/allocate.h
@@ -14,6 +14,9 @@
  * limitations under the License.
  */
 
+#ifndef _ALLOCATE_H_
+#define _ALLOCATE_H_
+
 #define EXT4_ALLOCATE_FAILED (u32)(~0)
 
 #include "ext4_utils.h"
@@ -49,3 +52,5 @@ void append_region(struct block_allocation *alloc,
        u32 block, u32 len, int bg);
 struct block_allocation *create_allocation();
 int append_oob_allocation(struct block_allocation *alloc, u32 len);
+
+#endif

diff --git a/ext4_utils/backed_block.c b/ext4_utils/backed_block.c
index c1a2f20..82c4bcc 100644 (file)
--- a/ext4_utils/backed_block.c
+++ b/ext4_utils/backed_block.c
@@ -14,17 +14,17 @@
  * limitations under the License.
  */
 
-#include <stdlib.h>
-
 #include "ext4_utils.h"
 #include "backed_block.h"
 
+#include <stdlib.h>
+
 struct data_block {
        u32 block;
        u32 len;
        u8 *data;
        const char *filename;
-       off_t offset;
+       off64_t offset;
        struct data_block *next;
 };
 
@@ -83,7 +83,7 @@ void queue_data_block(u8 *data, u32 len, u32 block)
 }
 
 /* Queues a chunk of a file on disk to be written to the specified data blocks */
-void queue_data_file(const char *filename, off_t offset, u32 len,
+void queue_data_file(const char *filename, off64_t offset, u32 len,
        u32 block)
 {
        struct data_block *db = malloc(sizeof(struct data_block));

diff --git a/ext4_utils/backed_block.h b/ext4_utils/backed_block.h
index 5304f00..61a1b1c 100644 (file)
--- a/ext4_utils/backed_block.h
+++ b/ext4_utils/backed_block.h
@@ -17,19 +17,17 @@
 #ifndef _BACKED_BLOCK_H_
 #define _BACKED_BLOCK_H_
 
-#include <sys/types.h>
-#include <unistd.h>
 #include "ext4_utils.h"
 #include "output_file.h"
 
 typedef void (*data_block_callback_t)(struct output_file *out, u64 off,
        u8 *data, int len);
 typedef void (*data_block_file_callback_t)(struct output_file *out, u64 off,
-                                          const char *file, off_t offset,
+                                          const char *file, off64_t offset,
                                           int len);
 
 void queue_data_block(u8 *data, u32 len, u32 block);
-void queue_data_file(const char *filename, off_t offset, u32 len,
+void queue_data_file(const char *filename, off64_t offset, u32 len,
        u32 block);
 void for_each_data_block(data_block_callback_t data_func,
        data_block_file_callback_t file_func, struct output_file *out);

diff --git a/ext4_utils/ext4_utils.c b/ext4_utils/ext4_utils.c
index 82d1116..1551633 100644 (file)
--- a/ext4_utils/ext4_utils.c
+++ b/ext4_utils/ext4_utils.c
@@ -14,6 +14,14 @@
  * limitations under the License.
  */
 
+#include "ext4_utils.h"
+#include "output_file.h"
+#include "backed_block.h"
+#include "uuid.h"
+#include "allocate.h"
+#include "indirect.h"
+#include "extent.h"
+
 #include <fcntl.h>
 #include <arpa/inet.h>
 #include <sys/ioctl.h>
@@ -27,14 +35,6 @@
 #include <sys/disk.h>
 #endif
 
-#include "ext4_utils.h"
-#include "output_file.h"
-#include "backed_block.h"
-#include "uuid.h"
-#include "allocate.h"
-#include "indirect.h"
-#include "extent.h"
-
 #include "ext4.h"
 #include "jbd2.h"
 
@@ -78,7 +78,6 @@ void write_ext4_image(const char *filename, int gz, int sparse)
 {
        int ret = 0;
        struct output_file *out = open_output_file(filename, gz, sparse);
-       off_t off;
 
        if (!out)
                return;

diff --git a/ext4_utils/ext4_utils.h b/ext4_utils/ext4_utils.h
index 712fea6..8a6c3c4 100644 (file)
--- a/ext4_utils/ext4_utils.h
+++ b/ext4_utils/ext4_utils.h
@@ -17,6 +17,12 @@
 #ifndef _EXT4_UTILS_H_
 #define _EXT4_UTILS_H_
 
+#define _GNU_SOURCE
+#define _FILE_OFFSET_BITS 64
+#define _LARGEFILE64_SOURCE
+#include <sys/types.h>
+#include <unistd.h>
+
 #include <sys/types.h>
 #include <errno.h>
 #include <stdarg.h>
@@ -24,6 +30,20 @@
 #include <stdlib.h>
 #include <string.h>
 
+#if defined(__APPLE__) && defined(__MACH__)
+#define lseek64 lseek
+#define off64_t off_t
+#endif
+
+#ifdef __BIONIC__
+extern void*  __mmap2(void *, size_t, int, int, int, off_t);
+static inline void *mmap64(void *addr, size_t length, int prot, int flags,
+        int fd, off64_t offset)
+{
+    return __mmap2(addr, length, prot, flags, fd, offset >> 12);
+}
+#endif
+
 extern int force;
 
 #define warn(fmt, args...) do { fprintf(stderr, "warning: %s: " fmt "\n", __func__, ## args); } while (0)

diff --git a/ext4_utils/extent.c b/ext4_utils/extent.c
index 1d2581d..daa2541 100644 (file)
--- a/ext4_utils/extent.c
+++ b/ext4_utils/extent.c
@@ -14,16 +14,16 @@
  * limitations under the License.
  */
 
-#include <stdlib.h>
-#include <stdio.h>
-
 #include "ext4_utils.h"
 #include "ext4.h"
 #include "ext4_extents.h"
 #include "backed_block.h"
-
 #include "extent.h"
 
+#include <stdlib.h>
+#include <stdio.h>
+
+
 /* Creates data buffers for the first backing_len bytes of a block allocation
    and queues them to be written */
 static u8 *extent_create_backing(struct block_allocation *alloc,
@@ -55,7 +55,7 @@ static u8 *extent_create_backing(struct block_allocation *alloc,
 static void extent_create_backing_file(struct block_allocation *alloc,
        u64 backing_len, const char *filename)
 {
-       off_t offset = 0;
+       off64_t offset = 0;
        for (; alloc != NULL && backing_len > 0; get_next_region(alloc)) {
                u32 region_block;
                u32 region_len;

diff --git a/ext4_utils/indirect.c b/ext4_utils/indirect.c
index d776d57..4e768a1 100644 (file)
--- a/ext4_utils/indirect.c
+++ b/ext4_utils/indirect.c
@@ -14,9 +14,6 @@
  * limitations under the License.
  */
 
-#include <stdlib.h>
-#include <stdio.h>
-
 #include "ext4_utils.h"
 #include "ext4.h"
 #include "ext4_extents.h"
@@ -24,6 +21,9 @@
 #include "indirect.h"
 #include "allocate.h"
 
+#include <stdlib.h>
+#include <stdio.h>
+
 /* Creates data buffers for the first backing_len bytes of a block allocation
    and queues them to be written */
 static u8 *create_backing(struct block_allocation *alloc,

diff --git a/ext4_utils/make_ext4fs.c b/ext4_utils/make_ext4fs.c
index d8eb1a3..cbec2ad 100644 (file)
--- a/ext4_utils/make_ext4fs.c
+++ b/ext4_utils/make_ext4fs.c
@@ -14,7 +14,12 @@
  * limitations under the License.
  */
 
-#define _GNU_SOURCE
+#include "make_ext4fs.h"
+#include "output_file.h"
+#include "ext4_utils.h"
+#include "allocate.h"
+#include "contents.h"
+#include "uuid.h"
 
 #include <dirent.h>
 #include <libgen.h>
@@ -25,13 +30,6 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 
-#include "make_ext4fs.h"
-#include "output_file.h"
-#include "ext4_utils.h"
-#include "allocate.h"
-#include "contents.h"
-#include "uuid.h"
-
 #ifdef ANDROID
 #include <private/android_filesystem_config.h>
 #endif

diff --git a/ext4_utils/output_file.c b/ext4_utils/output_file.c
index 3785b06..efb1307 100644 (file)
--- a/ext4_utils/output_file.c
+++ b/ext4_utils/output_file.c
@@ -13,7 +13,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#define _LARGEFILE64_SOURCE
+
+#include "ext4_utils.h"
+#include "output_file.h"
+#include "sparse_format.h"
+#include "sparse_crc32.h"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -24,11 +28,6 @@
 
 #include <zlib.h>
 
-#include "ext4_utils.h"
-#include "output_file.h"
-#include "sparse_format.h"
-#include "sparse_crc32.h"
-
 #if defined(__APPLE__) && defined(__MACH__)
 #define lseek64 lseek
 #define off64_t off_t
@@ -384,9 +383,11 @@ void write_data_block(struct output_file *out, u64 off, u8 *data, int len)
 
 /* Write a contiguous region of data blocks from a file */
 void write_data_file(struct output_file *out, u64 off, const char *file,
-                    off_t offset, int len)
+                    off64_t offset, int len)
 {
        int ret;
+       off64_t aligned_offset;
+       int aligned_diff;
 
        if (off + len >= info.len) {
                error("attempted to write block %llu past end of filesystem",
@@ -400,21 +401,25 @@ void write_data_file(struct output_file *out, u64 off, const char *file,
                return;
        }
 
-       u8 *data = mmap(NULL, len, PROT_READ, MAP_SHARED, file_fd, offset);
+       aligned_offset = offset & ~(4096 - 1);
+       aligned_diff = offset - aligned_offset;
+
+       u8 *data = mmap64(NULL, len + aligned_diff, PROT_READ, MAP_SHARED, file_fd,
+                       aligned_offset);
        if (data == MAP_FAILED) {
-               error_errno("mmap");
+               error_errno("mmap64");
                close(file_fd);
                return;
        }
 
        if (out->sparse) {
-               write_chunk_raw(out, off, data, len);
+               write_chunk_raw(out, off, data + aligned_diff, len);
        } else {
                ret = out->ops->seek(out, off);
                if (ret < 0)
                        goto err;
 
-               ret = out->ops->write(out, data, len);
+               ret = out->ops->write(out, data + aligned_diff, len);
                if (ret < 0)
                        goto err;
        }

diff --git a/ext4_utils/output_file.h b/ext4_utils/output_file.h
index 82b0952..393bc9b 100644 (file)
--- a/ext4_utils/output_file.h
+++ b/ext4_utils/output_file.h
@@ -14,11 +14,16 @@
  * limitations under the License.
  */
 
+#ifndef _OUTPUT_FILE_H_
+#define _OUTPUT_FILE_H_
+
 struct output_file;
 
 struct output_file *open_output_file(const char *filename, int gz, int sparse);
 void write_data_block(struct output_file *out, u64 off, u8 *data, int len);
 void write_data_file(struct output_file *out, u64 off, const char *file,
-                    off_t offset, int len);
+                    off64_t offset, int len);
 void pad_output_file(struct output_file *out, u64 len);
 void close_output_file(struct output_file *out);
+
+#endif

diff --git a/ext4_utils/simg2img.c b/ext4_utils/simg2img.c
index 88c67b4..9ef2509 100644 (file)
--- a/ext4_utils/simg2img.c
+++ b/ext4_utils/simg2img.c
@@ -13,8 +13,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#define _LARGEFILE64_SOURCE
-#define _FILE_OFFSET_BITS 64
+
+#include "ext4_utils.h"
+#include "output_file.h"
+#include "sparse_format.h"
+#include "sparse_crc32.h"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -24,11 +27,6 @@
 #include <fcntl.h>
 #include <stdio.h>
 
-#include "ext4_utils.h"
-#include "output_file.h"
-#include "sparse_format.h"
-#include "sparse_crc32.h"
-
 #if defined(__APPLE__) && defined(__MACH__)
 #define lseek64 lseek
 #define off64_t off_t