# HG changeset patch
# User Paul Boddie <paul@boddie.org.uk>
# Date 1653080076 -7200
# Node ID 8c1a864b44237ebba34a9aee6f114f74a0bdae25
# Parent  911d151bbbfcc70bb93c91f1999dccd38512549f
Introduced tentative support for masked file regions.

diff -r 911d151bbbfc -r 8c1a864b4423 libfsclient/include/fsclient/client.h
--- a/libfsclient/include/fsclient/client.h	Fri May 20 00:40:27 2022 +0200
+++ b/libfsclient/include/fsclient/client.h	Fri May 20 22:54:36 2022 +0200
@@ -64,7 +64,8 @@
 /* File and region operations. */
 
 long client_flush(file_t *file);
-void *client_mmap(file_t *file, offset_t position, offset_t length, l4re_rm_flags_t region_flags);
+void *client_mmap(file_t *file, offset_t position, offset_t length, int masked,
+                  l4re_rm_flags_t region_flags);
 l4re_rm_flags_t client_region_flags(prot_t prot, flags_t flags);
 
 /* Pipe region operations. */
diff -r 911d151bbbfc -r 8c1a864b4423 libfsclient/include/fsclient/file.h
--- a/libfsclient/include/fsclient/file.h	Fri May 20 00:40:27 2022 +0200
+++ b/libfsclient/include/fsclient/file.h	Fri May 20 22:54:36 2022 +0200
@@ -109,7 +109,8 @@
 /* File and region operations. */
 
 long file_flush(file_t *file);
-long file_mmap(file_t *file, offset_t position, offset_t length, l4re_rm_flags_t region_flags);
+long file_mmap(file_t *file, offset_t position, offset_t length, int masked,
+               l4re_rm_flags_t region_flags);
 l4re_rm_flags_t file_region_flags(flags_t flags);
 long file_resize(file_t *file, offset_t size);
 
diff -r 911d151bbbfc -r 8c1a864b4423 libfsclient/lib/src/client.cc
--- a/libfsclient/lib/src/client.cc	Fri May 20 00:40:27 2022 +0200
+++ b/libfsclient/lib/src/client.cc	Fri May 20 22:54:36 2022 +0200
@@ -81,7 +81,7 @@
 
     if ((position < file->start_pos) || (position >= file->end_pos))
     {
-      if (file_mmap(file, position, file_span(file),
+      if (file_mmap(file, position, file_span(file), 0,
                     file_region_flags(file->flags)))
         return -L4_EIO;
     }
@@ -191,7 +191,7 @@
   if (file->memory == NULL)
   {
     if (file->object_flags & OBJECT_SUPPORTS_MMAP)
-      return client_mmap(file, client_tell(file), count,
+      return client_mmap(file, client_tell(file), count, 0,
                          file_region_flags(file->flags));
     else if (pipe_current(file))
       return NULL;
@@ -487,10 +487,10 @@
 
 /* Map a memory region to a file. */
 
-void *client_mmap(file_t *file, offset_t position, offset_t length,
+void *client_mmap(file_t *file, offset_t position, offset_t length, int masked,
                   l4re_rm_flags_t region_flags)
 {
-  if ((file == NULL) || file_mmap(file, position, length, region_flags))
+  if ((file == NULL) || file_mmap(file, position, length, masked, region_flags))
     return NULL;
 
   return file->memory;
diff -r 911d151bbbfc -r 8c1a864b4423 libfsclient/lib/src/file.cc
--- a/libfsclient/lib/src/file.cc	Fri May 20 00:40:27 2022 +0200
+++ b/libfsclient/lib/src/file.cc	Fri May 20 22:54:36 2022 +0200
@@ -351,12 +351,12 @@
 /* Map a region of the given file to a memory region, obtaining an updated file
    size and populated data details. Unmap any previously mapped region. */
 
-long file_mmap(file_t *file, offset_t position, offset_t length,
+long file_mmap(file_t *file, offset_t position, offset_t length, int masked,
                l4re_rm_flags_t region_flags)
 {
   char *memory = file->memory;
   client_MappedFile mapped_file(file->ref);
-  long err = mapped_file.mmap(position, length, &file->start_pos,
+  long err = mapped_file.mmap(position, length, masked, &file->start_pos,
                               &file->end_pos, &file->size);
 
   if (err)
diff -r 911d151bbbfc -r 8c1a864b4423 libfsserver/include/fsserver/file_pager.h
--- a/libfsserver/include/fsserver/file_pager.h	Fri May 20 00:40:27 2022 +0200
+++ b/libfsserver/include/fsserver/file_pager.h	Fri May 20 22:54:36 2022 +0200
@@ -69,8 +69,8 @@
     virtual long map(offset_t offset, address_t hot_spot, map_flags_t flags,
                      l4_snd_fpage_t *region);
 
-    virtual long mmap(offset_t position, offset_t length, offset_t *start_pos,
-                      offset_t *end_pos, offset_t *size);
+    virtual long mmap(offset_t position, offset_t length, int masked,
+                      offset_t *start_pos, offset_t *end_pos, offset_t *size);
 
     /* Notification methods. */
 
diff -r 911d151bbbfc -r 8c1a864b4423 libfsserver/include/fsserver/pager.h
--- a/libfsserver/include/fsserver/pager.h	Fri May 20 00:40:27 2022 +0200
+++ b/libfsserver/include/fsserver/pager.h	Fri May 20 22:54:36 2022 +0200
@@ -1,7 +1,7 @@
 /*
  * Generic pager functionality.
  *
- * Copyright (C) 2021 Paul Boddie <paul@boddie.org.uk>
+ * Copyright (C) 2021, 2022 Paul Boddie <paul@boddie.org.uk>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -25,6 +25,7 @@
 
 #include <fsserver/page_mapper.h>
 #include <fsserver/resource.h>
+#include <mem/memory_incremental.h>
 
 
 
@@ -37,6 +38,20 @@
     PageMapper *_mapper;
     map_flags_t _flags;
 
+    /* Masked region support. */
+
+    bool _is_masked;
+    offset_t _start_visible, _end_visible;
+    Flexpage _start_flexpage, _end_flexpage, _zero_flexpage;
+    MemoryIncremental _memory;
+
+    Flexpage *get_masked_flexpage(Flexpage *flexpage);
+
+    void allocate_region(Flexpage *flexpage, Flexpage &masked);
+
+    void populate_region(Flexpage *flexpage, Flexpage &masked,
+                         bool has_start, bool has_end);
+
 public:
     explicit Pager(PageMapper *mapper, map_flags_t flags);
 
@@ -59,8 +74,8 @@
 
     /* Mapped file methods. */
 
-    virtual long mmap(offset_t position, offset_t length, offset_t *start_pos,
-                      offset_t *end_pos, offset_t *size);
+    virtual long mmap(offset_t position, offset_t length, int masked,
+                      offset_t *start_pos, offset_t *end_pos, offset_t *size);
 };
 
 // vim: tabstop=4 expandtab shiftwidth=4
diff -r 911d151bbbfc -r 8c1a864b4423 libfsserver/lib/files/file_pager.cc
--- a/libfsserver/lib/files/file_pager.cc	Fri May 20 00:40:27 2022 +0200
+++ b/libfsserver/lib/files/file_pager.cc	Fri May 20 22:54:36 2022 +0200
@@ -90,12 +90,12 @@
     return err;
 }
 
-long FilePager::mmap(offset_t position, offset_t length, offset_t *start_pos,
-                     offset_t *end_pos, offset_t *size)
+long FilePager::mmap(offset_t position, offset_t length, int masked,
+                     offset_t *start_pos, offset_t *end_pos, offset_t *size)
 {
     /* Set the limits of the paged region. */
 
-    return Pager::mmap(position, length, start_pos, end_pos, size);
+    return Pager::mmap(position, length, masked, start_pos, end_pos, size);
 }
 
 
diff -r 911d151bbbfc -r 8c1a864b4423 libfsserver/lib/generic/pager.cc
--- a/libfsserver/lib/generic/pager.cc	Fri May 20 00:40:27 2022 +0200
+++ b/libfsserver/lib/generic/pager.cc	Fri May 20 22:54:36 2022 +0200
@@ -1,7 +1,7 @@
 /*
  * Generic pager functionality.
  *
- * Copyright (C) 2021 Paul Boddie <paul@boddie.org.uk>
+ * Copyright (C) 2021, 2022 Paul Boddie <paul@boddie.org.uk>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -23,13 +23,16 @@
 #include "ipc.h"
 #include "pager.h"
 
+#include <string.h>
+
 
 
 /* Initialise the pager with a page mapper and the given flags controlling
    access to a file. */
 
 Pager::Pager(PageMapper *mapper, map_flags_t flags)
-: _start(0), _size(0), _mapper(mapper), _flags(flags)
+: _start(0), _size(0), _mapper(mapper), _flags(flags),
+  _is_masked(false), _start_visible(0), _end_visible(0)
 {
 }
 
@@ -61,8 +64,8 @@
 
 /* Expose a region of the file. */
 
-long Pager::mmap(offset_t position, offset_t length, offset_t *start_pos,
-                 offset_t *end_pos, offset_t *size)
+long Pager::mmap(offset_t position, offset_t length, int masked,
+                 offset_t *start_pos, offset_t *end_pos, offset_t *size)
 {
     /* Define region characteristics. */
 
@@ -75,6 +78,17 @@
     *end_pos = _start + _size;
     *size = _mapper->get_data_size();
 
+    /* Permit masking of mapped regions. */
+
+    if (masked)
+    {
+        _start_visible = position;
+        _end_visible = position + length;
+        _is_masked = (*start_pos != _start_visible) || (*end_pos != _end_visible);
+    }
+    else
+        _is_masked = false;
+
     return L4_EOK;
 }
 
@@ -87,17 +101,23 @@
     offset_t file_offset = _start + offset;
     offset_t max_offset = _start + _size;
 
-    /* Prevent access beyond that defined by the pager. */
+    /* Prevent access beyond that defined by the pager.
+       NOTE: Permitting executable requests here. This needs to be configured
+             when opening the pager or by another means. */
 
     if (flags & (~(_flags | L4RE_DS_F_X)))
         return -L4_EACCESS;
 
     Flexpage *flexpage = _mapper->get(file_offset, flags);
 
+    /* Determine if the flexpage should be masked. */
+
+    Flexpage *issued_flexpage = get_masked_flexpage(flexpage);
+
     /* Issue the flexpage via the IPC system. */
 
-    long err = ipc_prepare_flexpage(flexpage, file_offset, max_offset, hot_spot,
-                                    flags, region);
+    long err = ipc_prepare_flexpage(issued_flexpage, file_offset, max_offset,
+                                    hot_spot, flags, region);
 
     if (!err)
         err = complete_Dataspace_map(*region);
@@ -119,4 +139,93 @@
     return _mapper->get_data_size();
 }
 
+/* Detect flexpages with masked content, introducing separate flexpages
+   offering masked content from the same region. */
+
+Flexpage *Pager::get_masked_flexpage(Flexpage *flexpage)
+{
+    /* If not masking content, return the original flexpage. */
+
+    if (!_is_masked)
+        return flexpage;
+
+    /* Determine whether the flexpage involves the limits of the visible
+       region. */
+
+    bool has_start = flexpage->supports_position(_start_visible) &&
+                     flexpage->base_offset != _start_visible;
+    bool has_end = flexpage->supports_position(_end_visible);
+    bool has_zero = (flexpage->base_offset >= _end_visible) ||
+                    (flexpage->base_offset + flexpage->size < _start_visible);
+
+    /* Return the original flexpage within the visible limits. */
+
+    if (!has_start && !has_end && !has_zero)
+        return flexpage;
+
+    /* Allocate and populate a region in one of the preallocated flexpages for
+       masked content.
+       NOTE: A general copy-on-write solution will maintain a collection of
+             flexpages. */
+
+    Flexpage &masked = has_start ? _start_flexpage :
+                       has_end ? _end_flexpage :
+                                 _zero_flexpage;
+
+    allocate_region(flexpage, masked);
+    populate_region(flexpage, masked, has_start, has_end);
+
+    /* Associate the preallocated flexpage with the original flexpage. */
+
+    flexpage->associate(&masked);
+    return &masked;
+}
+
+void Pager::allocate_region(Flexpage *flexpage, Flexpage &masked)
+{
+    offset_t needed = flexpage->region->size();
+
+    /* Attempt to re-use an existing region. */
+
+    if (masked.region != NULL)
+    {
+        if (masked.region->size() == needed)
+            return;
+        else
+            _memory.release(masked.region);
+    }
+
+    /* Set the region in the flexpage. */
+
+    masked.set_region(_memory.region(needed));
+    masked.reset(flexpage->base_offset);
+}
+
+void Pager::populate_region(Flexpage *flexpage, Flexpage &masked,
+                            bool has_start, bool has_end)
+{
+    /* Without any limits involved, provide a zero flexpage. */
+
+    if (!has_start && !has_end)
+    {
+        memset((void *) masked.region->start, 0, masked.size);
+        return;
+    }
+
+    /* Determine the content limits given start and/or end offsets. */
+
+    offset_t start_offset = has_start ? _start_visible - masked.base_offset : 0;
+    offset_t end_offset = has_end ? _end_visible - masked.base_offset : masked.size;
+
+    if (has_start)
+        memset((void *) masked.region->start, 0, start_offset);
+
+    memcpy((void *) (masked.region->start + start_offset),
+           (const void *) (flexpage->region->start + start_offset),
+           end_offset - start_offset);
+
+    if (has_end)
+        memset((void *) (masked.region->start + end_offset), 0, masked.size - end_offset);
+}
+
 // vim: tabstop=4 expandtab shiftwidth=4
diff -r 911d151bbbfc -r 8c1a864b4423 libfsserver/lib/mapping/page_mapper.cc
--- a/libfsserver/lib/mapping/page_mapper.cc	Fri May 20 00:40:27 2022 +0200
+++ b/libfsserver/lib/mapping/page_mapper.cc	Fri May 20 22:54:36 2022 +0200
@@ -162,11 +162,31 @@
 {
     if (flexpage->decrement() || purge)
     {
+        /* NOTE: Derived flexpages might potentially support their contents
+                 being merged into the flushed data, although this is a
+                 non-trivial problem. */
+
         if (flexpage->modified())
             _accessor->flush(flexpage);
 
+        /* Unmap the flexpage, requiring users of its memory to request another
+           flexpage in future. */
+
         ipc_unmap_flexpage(flexpage);
         flexpage->invalidate();
+
+        /* Unmap all derived flexpages, since these rely on the underlying file
+           contents. */
+
+        DerivedFlexpages::iterator it;
+
+        for (it = flexpage->derived.begin(); it != flexpage->derived.end(); it++)
+        {
+            ipc_unmap_flexpage(*it);
+            (*it)->invalidate();
+        }
+
+        flexpage->disassociate();
     }
 }
 
diff -r 911d151bbbfc -r 8c1a864b4423 libmem/include/mem/flexpage.h
--- a/libmem/include/mem/flexpage.h	Fri May 20 00:40:27 2022 +0200
+++ b/libmem/include/mem/flexpage.h	Fri May 20 22:54:36 2022 +0200
@@ -1,7 +1,7 @@
 /*
  * A flexpage abstraction.
  *
- * Copyright (C) 2021 Paul Boddie <paul@boddie.org.uk>
+ * Copyright (C) 2021, 2022 Paul Boddie <paul@boddie.org.uk>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -21,6 +21,8 @@
 
 #pragma once
 
+#include <list>
+
 #include <mem/memory_utils.h>
 #include <mem/region.h>
 #include <mem/send_flexpage.h>
@@ -34,6 +36,16 @@
 
 
 
+/* Forward declaration. */
+
+class Flexpage;
+
+/* Employed data types. */
+
+typedef std::list<Flexpage *> DerivedFlexpages;
+
+
+
 /* A flexpage abstraction. */
 
 class Flexpage
@@ -45,6 +57,10 @@
 public:
     Region *region;
 
+    /* Flexpage relationships to support masked regions. */
+
+    DerivedFlexpages derived;
+
     /* General flexpage characteristics. */
 
     offset_t base_addr, size;
@@ -56,10 +72,13 @@
 
     /* Associate a flexpage with a memory 'region'. */
 
-    explicit Flexpage(Region *region) : region(region)
+    explicit Flexpage(Region *region = NULL)
+    : region(region)
     {
     }
 
+    void set_region(Region *region);
+
     void reset(offset_t offset);
 
     bool decrement();
@@ -80,6 +99,12 @@
                          offset_t max_offset=0);
 
     SendFlexpage to_unmap();
+
+    /* Associate other flexpages with this flexpage. */
+
+    void associate(Flexpage *flexpage);
+
+    void disassociate();
 };
 
 // vim: tabstop=4 expandtab shiftwidth=4
diff -r 911d151bbbfc -r 8c1a864b4423 libmem/lib/src/flexpage.cc
--- a/libmem/lib/src/flexpage.cc	Fri May 20 00:40:27 2022 +0200
+++ b/libmem/lib/src/flexpage.cc	Fri May 20 22:54:36 2022 +0200
@@ -1,7 +1,7 @@
 /*
  * A flexpage abstraction.
  *
- * Copyright (C) 2021 Paul Boddie <paul@boddie.org.uk>
+ * Copyright (C) 2021, 2022 Paul Boddie <paul@boddie.org.uk>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -31,6 +31,9 @@
 
 void Flexpage::reset(offset_t offset)
 {
+    if (region == NULL)
+        return;
+
     _counter = 0;
 
     /* By definition (see "Flexible-Sized Page Objects - Object-Orientation
@@ -63,6 +66,13 @@
     page_addr = base_addr + page_offset;
 }
 
+/* Set a region. */
+
+void Flexpage::set_region(Region *region)
+{
+    this->region = region;
+}
+
 /* Decrement the usage counter, returning whether the flexpage is now no longer
    used. */
 
@@ -179,4 +189,16 @@
     return SendFlexpage(base_addr, page_order(size), _flags);
 }
 
+/* Associate another flexpage with this flexpage. */
+
+void Flexpage::associate(Flexpage *flexpage)
+{
+    derived.push_back(flexpage);
+}
+
+void Flexpage::disassociate()
+{
+    derived.clear();
+}
+
 // vim: tabstop=4 expandtab shiftwidth=4
diff -r 911d151bbbfc -r 8c1a864b4423 libsystypes/idl/mapped_file.idl
--- a/libsystypes/idl/mapped_file.idl	Fri May 20 00:40:27 2022 +0200
+++ b/libsystypes/idl/mapped_file.idl	Fri May 20 22:54:36 2022 +0200
@@ -4,10 +4,12 @@
 
 interface MappedFile
 {
-  /* Memory-map a file for the given file position and length, obtaining the
-     limits of the mapped region and the size of the file. */
+  /* Memory-map a file for the given file position and length, masking out of
+     range regions if indicated, obtaining the limits of the mapped region and
+     the size of the file. */
 
   [opcode(7)] void mmap(in offset_t position, in offset_t length,
+                        in int masked,
                         out offset_t start_pos, out offset_t end_pos,
                         out offset_t size);
 };
diff -r 911d151bbbfc -r 8c1a864b4423 tests/dstest_block_client.cc
--- a/tests/dstest_block_client.cc	Fri May 20 00:40:27 2022 +0200
+++ b/tests/dstest_block_client.cc	Fri May 20 22:54:36 2022 +0200
@@ -45,9 +45,24 @@
     unsigned long sample_remaining = remaining < sample ? remaining : sample;
 
     printf("%ld bytes from %p...\n", sample_remaining, (file->memory + offset));
-    strncpy(buf, (file->memory + offset), sample_remaining);
+    memcpy(buf, (file->memory + offset), sample_remaining);
     buf[sample_remaining] = '\0';
-    printf("%s\n", buf);
+
+    unsigned long leading = 0;
+    char *outbuf = buf;
+
+    while ((*outbuf == '\0') && (leading < sample_remaining))
+    {
+      outbuf++;
+      leading++;
+    }
+
+    if (leading)
+      printf("[%ld zero bytes]\n", leading);
+
+    printf("%s\n", outbuf);
+
+    printf("[%ld bytes after string]\n", sample_remaining - leading - strlen(outbuf));
   }
 }
 
@@ -80,9 +95,10 @@
     return 1;
   }
 
-  /* A region of the file is mapped. */
+  /* A region of the file is mapped. Here, the start and length will not provide
+     page-aligned offsets, but the region is nevertheless not masked. */
 
-  err = file_mmap(&file, 0, page(10), file_region_flags(file.flags));
+  err = file_mmap(&file, 10, 290, 0, file_region_flags(file.flags));
 
   if (err)
   {
@@ -90,16 +106,39 @@
     return 1;
   }
 
+  printf("File contents:\n");
+
   show(&file, step, sample);
 
+  printf("File shown.\n");
+
+  /* A region of the file is mapped. Here, the resulting offsets will be used to
+     define a masked region. */
+
+  err = file_mmap(&file, 10, 290, 1, file_region_flags(file.flags));
+
+  if (err)
+  {
+    printf("Could not map file region: %s\n", l4sys_errtostr(err));
+    return 1;
+  }
+
+  printf("File contents:\n");
+
+  show(&file, step, sample);
+
+  printf("File shown.\n");
+
   /* Resizing must occur before writing beyond the end of file. Otherwise, the
      data may get discarded if the supporting flexpage needs to be flushed. */
 
-  offset_t new_region = round(file_populated_span(&file), page(1));
+  offset_t old_size = file_populated_span(&file);
+  offset_t new_region = round(old_size, page(1));
+  offset_t new_size = new_region + old_size;
 
-  printf("Resize to %ld...\n", new_region + file_populated_span(&file));
+  printf("Resize to %ld...\n", new_size);
 
-  err = file_resize(&file, new_region + file_populated_span(&file));
+  err = file_resize(&file, new_size);
 
   if (err)
   {
@@ -109,17 +148,46 @@
 
   printf("Resized file...\n");
 
+  /* Re-map to avoid masking the region. */
+
+  err = file_mmap(&file, 10, new_size - 20, 0, file_region_flags(file.flags));
+
+  if (err)
+  {
+    printf("Could not map file region: %s\n", l4sys_errtostr(err));
+    return 1;
+  }
+
   /* Copy the sampled data to another file region. */
 
   printf("Copy data to %ld...\n", new_region);
 
-  for (unsigned long offset = 0; offset < file_populated_span(&file); offset += step)
+  for (unsigned long offset = 0; offset < old_size; offset += step)
   {
+    printf("Copying to %ld...\n", new_region + offset);
     memcpy(file.memory + new_region + offset, file.memory + offset, sample);
     if (step > sample)
       memset(file.memory + new_region + offset + sample, 0, step - sample);
   }
 
+  printf("File contents:\n");
+
+  show(&file, step, sample);
+
+  printf("File shown.\n");
+
+  /* Re-map to mask the region again. */
+
+  err = file_mmap(&file, 10, new_size - 20, 1, file_region_flags(file.flags));
+
+  if (err)
+  {
+    printf("Could not map file region: %s\n", l4sys_errtostr(err));
+    return 1;
+  }
+
+  printf("File contents:\n");
+
   show(&file, step, sample);
 
   printf("File shown.\n");
diff -r 911d151bbbfc -r 8c1a864b4423 tests/dstest_host_client.cc
--- a/tests/dstest_host_client.cc	Fri May 20 00:40:27 2022 +0200
+++ b/tests/dstest_host_client.cc	Fri May 20 22:54:36 2022 +0200
@@ -68,7 +68,7 @@
 
   /* A region of the file is mapped. */
 
-  err = file_mmap(&file, 0, page(10), file_region_flags(file.flags));
+  err = file_mmap(&file, 0, page(10), 0, file_region_flags(file.flags));
 
   if (err)
   {
diff -r 911d151bbbfc -r 8c1a864b4423 tests/dstest_test_client.cc
--- a/tests/dstest_test_client.cc	Fri May 20 00:40:27 2022 +0200
+++ b/tests/dstest_test_client.cc	Fri May 20 22:54:36 2022 +0200
@@ -77,7 +77,7 @@
 
   /* A region of the file is mapped. */
 
-  err = file_mmap(&file, page(start_page), page(MAP_PAGES), file_region_flags(file.flags));
+  err = file_mmap(&file, page(start_page), page(MAP_PAGES), 0, file_region_flags(file.flags));
 
   if (err)
   {