Re: I: coreutils tests/misc/nproc-avail fails on GNU/Linux without /proc and /sys mounted

[Bruno Haible]

Pádraig Brady wrote:
> Just to summarize what's happening here...
> 
> There are 3 CPU counts possible:
> 
> total >= online >= available
> 
> "online" corresponds to the CPUs enabled system wide,
> whereas "available" is what's available to a particular
> process which may be less due to affinity config.
> 
> "online" is not currently exposed through nproc
> but for reference is got on linux using:
> $ strace -e open getconf _NPROCESSORS_ONLN
>    open("/proc/stat"
>      for_each_online_cpu() //available to scheduler
> 
> "available" on linux is determined using:
> nproc
>    glibc::sched_getaffinity()
>      return corresponding_syscall();
> 
> "total" on linux is determined using:
> nproc --all
>    glibc::sysconf(_NPROCESSORS_CONF)
>      __get_nprocs_conf ()
>      {
>        int result = 1;
>        /* XXX Here will come a test for the new system call.  */
>        if (open("/sys/devices/system/cpu"))
>          result = parse();
>        else if (open("/proc/cpuinfo"))
>          result = parse();
>        return result;
>      }
> 
> The last one above is giving the issue as
> it relies on /sys or /proc being available.

Ah! Thanks for the detailed explanation.

> So what about a possible work around?
> How about doing this in nproc(1):
> 
> if (num_processors(NPROC_ALL) == 1)
>      return num_processors(NPROC_CURRENT_OVERRIDABLE)

It should be NPROC_CURRENT, not NPROC_CURRENT_OVERRIDABLE, I think,
because NPROC_CURRENT_OVERRIDABLE can be overridden by the user quite
arbitrarily.

I like this workaround, but would prefer to have it in the nproc module
in gnulib. coreutils/src/nproc.c is purely the command-line program.

Here's a proposed patch.


2010-01-10  Bruno Haible  <address@hidden>

        nproc: Work better on Linux when /proc and /sys are not mounted.
        * lib/nproc.c (num_processors_via_affinity_mask): New function,
        extracted from num_processors.
        (num_processors): Call it. Use it as lower bound when, on glibc/Linux
        systems, sysconf (_SC_NPROCESSORS_CONF) returns 1.
        Suggested by Pádraig Brady <address@hidden>.
        Reported by Dmitry V. Levin <address@hidden>.

*** lib/nproc.c.orig    Sun Jan 10 11:24:18 2010
--- lib/nproc.c Sun Jan 10 11:21:40 2010
***************
*** 59,229 ****
  
  #define ARRAY_SIZE(a) (sizeof (a) / sizeof ((a)[0]))
  
! unsigned long int
! num_processors (enum nproc_query query)
  {
!   if (query == NPROC_CURRENT_OVERRIDABLE)
!     {
!       /* Test the environment variable OMP_NUM_THREADS, recognized also by all
!          programs that are based on OpenMP.  The OpenMP spec says that the
!          value assigned to the environment variable "may have leading and
!          trailing white space". */
!       const char *envvalue = getenv ("OMP_NUM_THREADS");
! 
!       if (envvalue != NULL)
!         {
!           while (*envvalue != '\0' && c_isspace (*envvalue))
!             envvalue++;
!           /* Convert it from decimal to 'unsigned long'.  */
!           if (c_isdigit (*envvalue))
!             {
!               char *endptr = NULL;
!               unsigned long int value = strtoul (envvalue, &endptr, 10);
! 
!               if (endptr != NULL)
!                 {
!                   while (*endptr != '\0' && c_isspace (*endptr))
!                     endptr++;
!                   if (*endptr == '\0')
!                     return (value > 0 ? value : 1);
!                 }
!             }
!         }
! 
!       query = NPROC_CURRENT;
!     }
!   /* Here query is one of NPROC_ALL, NPROC_CURRENT.  */
! 
!   if (query == NPROC_CURRENT)
!     {
!       /* glibc >= 2.3.3 with NPTL and NetBSD 5 have pthread_getaffinity_np,
!          but with different APIs.  Also it requires linking with -lpthread.
!          Therefore this code is not enabled.
!          glibc >= 2.3.4 has sched_getaffinity whereas NetBSD 5 has
!          sched_getaffinity_np.  */
  #if HAVE_PTHREAD_AFFINITY_NP && defined __GLIBC__ && 0
!       {
!         cpu_set_t set;
  
!         if (pthread_getaffinity_np (pthread_self (), sizeof (set), &set) == 0)
!           {
!             unsigned long count;
  
  # ifdef CPU_COUNT
!             /* glibc >= 2.6 has the CPU_COUNT macro.  */
!             count = CPU_COUNT (&set);
  # else
!             size_t i;
  
!             count = 0;
!             for (i = 0; i < CPU_SETSIZE; i++)
!               if (CPU_ISSET (i, &set))
!                 count++;
  # endif
!             if (count > 0)
!               return count;
!           }
        }
  #elif HAVE_PTHREAD_AFFINITY_NP && defined __NetBSD__ && 0
        {
!         cpuset_t *set;
  
!         set = cpuset_create ();
!         if (set != NULL)
            {
!             unsigned long count = 0;
  
!             if (pthread_getaffinity_np (pthread_self (), cpuset_size (set), 
set)
!                 == 0)
                {
!                 cpuid_t i;
! 
!                 for (i = 0;; i++)
!                   {
!                     int ret = cpuset_isset (i, set);
!                     if (ret < 0)
!                       break;
!                     if (ret > 0)
!                       count++;
!                   }
                }
-             cpuset_destroy (set);
-             if (count > 0)
-               return count;
            }
        }
  #elif HAVE_SCHED_GETAFFINITY_LIKE_GLIBC /* glibc >= 2.3.4 */
!       {
!         cpu_set_t set;
  
!         if (sched_getaffinity (0, sizeof (set), &set) == 0)
!           {
!             unsigned long count;
  
  # ifdef CPU_COUNT
!             /* glibc >= 2.6 has the CPU_COUNT macro.  */
!             count = CPU_COUNT (&set);
  # else
!             size_t i;
  
!             count = 0;
!             for (i = 0; i < CPU_SETSIZE; i++)
!               if (CPU_ISSET (i, &set))
!                 count++;
  # endif
!             if (count > 0)
!               return count;
!           }
        }
  #elif HAVE_SCHED_GETAFFINITY_NP /* NetBSD >= 5 */
        {
!         cpuset_t *set;
  
!         set = cpuset_create ();
!         if (set != NULL)
            {
!             unsigned long count = 0;
  
!             if (sched_getaffinity_np (getpid (), cpuset_size (set), set) == 0)
                {
!                 cpuid_t i;
! 
!                 for (i = 0;; i++)
!                   {
!                     int ret = cpuset_isset (i, set);
!                     if (ret < 0)
!                       break;
!                     if (ret > 0)
!                       count++;
!                   }
                }
-             cpuset_destroy (set);
-             if (count > 0)
-               return count;
            }
        }
  #endif
  
  #if (defined _WIN32 || defined __WIN32__) && ! defined __CYGWIN__
!       { /* This works on native Windows platforms.  */
!         DWORD_PTR process_mask;
!         DWORD_PTR system_mask;
  
!         if (GetProcessAffinityMask (GetCurrentProcess (),
!                                     &process_mask, &system_mask))
!           {
!             DWORD_PTR mask = process_mask;
!             unsigned long count = 0;
  
!             for (; mask != 0; mask = mask >> 1)
!               if (mask & 1)
!                 count++;
!             if (count > 0)
!               return count;
!           }
        }
  #endif
  
  #if defined _SC_NPROCESSORS_ONLN
        { /* This works on glibc, MacOS X 10.5, FreeBSD, AIX, OSF/1, Solaris,
             Cygwin, Haiku.  */
--- 59,261 ----
  
  #define ARRAY_SIZE(a) (sizeof (a) / sizeof ((a)[0]))
  
! /* Return the number of processors available to the current process, based
!    on a modern system call that returns the "affinity" between the current
!    process and each CPU.  Return 0 if unknown or if such a system call does
!    not exist.  */
! static unsigned long
! num_processors_via_affinity_mask (void)
  {
!   /* glibc >= 2.3.3 with NPTL and NetBSD 5 have pthread_getaffinity_np,
!      but with different APIs.  Also it requires linking with -lpthread.
!      Therefore this code is not enabled.
!      glibc >= 2.3.4 has sched_getaffinity whereas NetBSD 5 has
!      sched_getaffinity_np.  */
  #if HAVE_PTHREAD_AFFINITY_NP && defined __GLIBC__ && 0
!   {
!     cpu_set_t set;
  
!     if (pthread_getaffinity_np (pthread_self (), sizeof (set), &set) == 0)
!       {
!         unsigned long count;
  
  # ifdef CPU_COUNT
!         /* glibc >= 2.6 has the CPU_COUNT macro.  */
!         count = CPU_COUNT (&set);
  # else
!         size_t i;
  
!         count = 0;
!         for (i = 0; i < CPU_SETSIZE; i++)
!           if (CPU_ISSET (i, &set))
!             count++;
  # endif
!         if (count > 0)
!           return count;
        }
+   }
  #elif HAVE_PTHREAD_AFFINITY_NP && defined __NetBSD__ && 0
+   {
+     cpuset_t *set;
+ 
+     set = cpuset_create ();
+     if (set != NULL)
        {
!         unsigned long count = 0;
  
!         if (pthread_getaffinity_np (pthread_self (), cpuset_size (set), set)
!             == 0)
            {
!             cpuid_t i;
  
!             for (i = 0;; i++)
                {
!                 int ret = cpuset_isset (i, set);
!                 if (ret < 0)
!                   break;
!                 if (ret > 0)
!                   count++;
                }
            }
+         cpuset_destroy (set);
+         if (count > 0)
+           return count;
        }
+   }
  #elif HAVE_SCHED_GETAFFINITY_LIKE_GLIBC /* glibc >= 2.3.4 */
!   {
!     cpu_set_t set;
  
!     if (sched_getaffinity (0, sizeof (set), &set) == 0)
!       {
!         unsigned long count;
  
  # ifdef CPU_COUNT
!         /* glibc >= 2.6 has the CPU_COUNT macro.  */
!         count = CPU_COUNT (&set);
  # else
!         size_t i;
  
!         count = 0;
!         for (i = 0; i < CPU_SETSIZE; i++)
!           if (CPU_ISSET (i, &set))
!             count++;
  # endif
!         if (count > 0)
!           return count;
        }
+   }
  #elif HAVE_SCHED_GETAFFINITY_NP /* NetBSD >= 5 */
+   {
+     cpuset_t *set;
+ 
+     set = cpuset_create ();
+     if (set != NULL)
        {
!         unsigned long count = 0;
  
!         if (sched_getaffinity_np (getpid (), cpuset_size (set), set) == 0)
            {
!             cpuid_t i;
  
!             for (i = 0;; i++)
                {
!                 int ret = cpuset_isset (i, set);
!                 if (ret < 0)
!                   break;
!                 if (ret > 0)
!                   count++;
                }
            }
+         cpuset_destroy (set);
+         if (count > 0)
+           return count;
        }
+   }
  #endif
  
  #if (defined _WIN32 || defined __WIN32__) && ! defined __CYGWIN__
!   { /* This works on native Windows platforms.  */
!     DWORD_PTR process_mask;
!     DWORD_PTR system_mask;
  
!     if (GetProcessAffinityMask (GetCurrentProcess (),
!                                 &process_mask, &system_mask))
!       {
!         DWORD_PTR mask = process_mask;
!         unsigned long count = 0;
  
!         for (; mask != 0; mask = mask >> 1)
!           if (mask & 1)
!             count++;
!         if (count > 0)
!           return count;
        }
+   }
  #endif
  
+   return 0;
+ }
+ 
+ unsigned long int
+ num_processors (enum nproc_query query)
+ {
+   if (query == NPROC_CURRENT_OVERRIDABLE)
+     {
+       /* Test the environment variable OMP_NUM_THREADS, recognized also by all
+          programs that are based on OpenMP.  The OpenMP spec says that the
+          value assigned to the environment variable "may have leading and
+          trailing white space". */
+       const char *envvalue = getenv ("OMP_NUM_THREADS");
+ 
+       if (envvalue != NULL)
+         {
+           while (*envvalue != '\0' && c_isspace (*envvalue))
+             envvalue++;
+           /* Convert it from decimal to 'unsigned long'.  */
+           if (c_isdigit (*envvalue))
+             {
+               char *endptr = NULL;
+               unsigned long int value = strtoul (envvalue, &endptr, 10);
+ 
+               if (endptr != NULL)
+                 {
+                   while (*endptr != '\0' && c_isspace (*endptr))
+                     endptr++;
+                   if (*endptr == '\0')
+                     return (value > 0 ? value : 1);
+                 }
+             }
+         }
+ 
+       query = NPROC_CURRENT;
+     }
+   /* Here query is one of NPROC_ALL, NPROC_CURRENT.  */
+ 
+   /* On systems with a modern affinity mask system call, we have
+          sysconf (_SC_NPROCESSORS_CONF)
+             >= sysconf (_SC_NPROCESSORS_ONLN)
+                >= num_processors_via_affinity_mask ()
+      The first number is the number of CPUs configured in the system.
+      The second number is the number of CPUs available to the scheduler.
+      The third number is the number of CPUs available to the current process.
+ 
+      Note! On Linux systems with glibc, the first and second number come from
+      the /sys and /proc file systems (see
+      glibc/sysdeps/unix/sysv/linux/getsysstats.c).
+      In some situations these file systems are not mounted, and the sysconf
+      call returns 1, which does not reflect the reality.  */
+ 
+   if (query == NPROC_CURRENT)
+     {
+       /* Try the modern affinity mask system call.  */
+       {
+         unsigned long nprocs = num_processors_via_affinity_mask ();
+ 
+         if (nprocs > 0)
+           return nprocs;
+       }
+ 
  #if defined _SC_NPROCESSORS_ONLN
        { /* This works on glibc, MacOS X 10.5, FreeBSD, AIX, OSF/1, Solaris,
             Cygwin, Haiku.  */
***************
*** 239,244 ****
--- 271,292 ----
        { /* This works on glibc, MacOS X 10.5, FreeBSD, AIX, OSF/1, Solaris,
             Cygwin, Haiku.  */
          long int nprocs = sysconf (_SC_NPROCESSORS_CONF);
+ 
+ # if __GLIBC__ >= 2 && defined __linux__
+         /* On Linux systems with glibc, this information comes from the /sys 
and
+            /proc file systems (see 
glibc/sysdeps/unix/sysv/linux/getsysstats.c).
+            In some situations these file systems are not mounted, and the
+            sysconf call returns 1.  But we wish to guarantee that
+            num_processors (NPROC_ALL) >= num_processors (NPROC_CURRENT).  */
+         if (nprocs == 1)
+           {
+             unsigned long nprocs_current = num_processors_via_affinity_mask 
();
+ 
+             if (nprocs_current > 0)
+               nprocs = nprocs_current;
+           }
+ # endif
+ 
          if (nprocs > 0)
            return nprocs;
        }