diff options
| -rw-r--r-- | Makefile.am | 3 | ||||
| -rwxr-xr-x | scripts/system/irq-affinity.pl | 289 | ||||
| -rwxr-xr-x | scripts/system/vyatta-auto-irqaffin.pl | 467 | ||||
| -rwxr-xr-x | scripts/system/vyatta-irqaffin | 194 | ||||
| -rw-r--r-- | templates/interfaces/ethernet/node.tag/smp_affinity/node.def | 18 |
5 files changed, 294 insertions, 677 deletions
diff --git a/Makefile.am b/Makefile.am index d9c35d8a..f349121e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -32,8 +32,7 @@ sbin_SCRIPTS += scripts/system/vyatta_update_syslog.pl sbin_SCRIPTS += scripts/system/vyatta_update_console.pl sbin_SCRIPTS += scripts/system/vyatta_update_ntp.pl sbin_SCRIPTS += scripts/system/vyatta_update_telnet -sbin_SCRIPTS += scripts/system/vyatta-auto-irqaffin.pl -sbin_SCRIPTS += scripts/system/vyatta-irqaffin +sbin_SCRIPTS += scripts/system/irq-affinity.pl sbin_SCRIPTS += scripts/snmp/vyatta-snmp.pl sbin_SCRIPTS += scripts/snmp/snmpd.init sbin_SCRIPTS += scripts/snmp/if-mib-alias diff --git a/scripts/system/irq-affinity.pl b/scripts/system/irq-affinity.pl new file mode 100755 index 00000000..eaad95cc --- /dev/null +++ b/scripts/system/irq-affinity.pl @@ -0,0 +1,289 @@ +#!/usr/bin/perl + +# **** License **** +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# This code was originally developed by Vyatta, Inc. +# Portions created by Vyatta are Copyright (C) 2009,2010 Vyatta, Inc. +# All Rights Reserved. +# +# **** End License **** +# +use warnings; +use strict; +use Sys::Syslog qw(:standard :macros); + +die "Usage: $0 ifname {auto | mask}\n" if ($#ARGV < 1); + +my ($ifname, $mask) = @ARGV; + +die "Error: Interface $ifname does not exist\n" + unless -d "/sys/class/net/$ifname"; + +openlog("irq-affinity","",LOG_LOCAL0); + +my ( $cpus, $cores ) = cpuinfo(); + +if ($mask eq 'auto') { + affinity_auto($ifname); +} else { + affinity_mask($ifname, $mask); +} + +exit 0; + +# Get current irq assignments by reading /proc/interrupts +sub irqinfo { + my $irqmap; + + open( my $f, '<', "/proc/interrupts" ) + or die "Can't read /proc/interrupts"; + + while (<$f>) { + chomp; + my @cols = split; + + # First column is IRQ number (and colon) + next unless /^\s*(\d+):\s/; + my $irq = $1; + + # Skip columns for IRQ's per CPU + foreach my $name ( @cols[ $cpus+1 .. $#cols ] ) { + $name =~ s/,$//; + $irqmap->{$name} = $irq; + } + } + close $f; + + return $irqmap; +} + +# Determine number of cpus and cores +sub cpuinfo { + my ( $cpu, $core ); + + open( my $f, '<', "/proc/cpuinfo" ) + or die "Can't read /proc/cpuinfo"; + + while (<$f>) { + chomp; + if (/^cpu cores\s+:\s(\d+)$/) { + $core = $1; + } + elsif (/^processor\s+:\s+(\d)$/) { + $cpu = $1; + } + } + close $f; + + return ( $cpu + 1, $core ); +} + +# Set affinity value for a irq +sub set_affinity { + my ( $ifname, $irq, $mask ) = @_; + my $smp_affinity = "/proc/irq/$irq/smp_affinity"; + + syslog(LOG_INFO, "%s: irq %d affinity set to 0x%x", $ifname, $irq, $mask); + + open( my $f, '>', $smp_affinity ) + or die "Can't open: $smp_affinity : $!\n"; + printf {$f} "%x\n", $mask; + close $f; +} + +# set Receive Packet Steering mask +sub set_rps { + my ( $ifname, $q, $mask ) = @_; + + # ignore if older kernel without RPS + my $rxq = "/sys/class/net/$ifname/queues"; + return unless ( -d $rxq ); + + syslog(LOG_INFO, "%s: receive queue %d cpus set to 0x%x", + $ifname, $q, $mask); + + my $rps_cpus = "$rxq/rx-$q/rps_cpus"; + open( my $f, '>', $rps_cpus ) + or die "Can't open: $rps_cpus : $!\n"; + printf {$f} "%x\n", $mask; + close $f; +} + +# For multi-queue NIC choose next cpu to be on next core +# FIXME assumes all cpu's online +sub next_cpu { + my $cpu = shift; + my $threads = $cpus / $cores; # threads per core + + $cpu += $threads; + if ( $cpu >= $cpus ) { + $cpu = ($cpu + 1) % $threads; # next thread on core 0 + } + return $cpu; +} + +# First cpu to assign for the queues +sub first_cpu { + my ($ifname, $numq) = @_; + + # For multi-queue nic's always starts with 0 + # This is less than ideal when there are more core's available + # than number of queues (probably should barber pole); + # but the Intel IXGBE needs CPU 0 <-> queue 0 because of flow director + return 0 if ($numq > 1); + + # For single-queue nic choose IRQ based on name + # Ideally should make decision on least loaded CPU + my ($ifunit) = ($ifname =~ m/^[a-z]*(\d+)$/); + die "can't find number for $ifname\n" + unless defined($ifunit); + + return ( $ifunit * ($cpus / $cores) ) % $cpus; +} + +# Assignment for multi-queue NICs +# Assign each queue to successive cores +sub assign_multiqueue { + my ( $ifname, $numq, $irqmap, $irqfmt ) = @_; + my $cpu = 0; + + for ( my $q = first_cpu($ifname, $numq) ; $q < $numq ; $q++ ) { + # handles multiple irq's per interface (tx/rx) + foreach my $fmt (@$irqfmt) { + my $name = sprintf( $fmt, $ifname, $q ); + my $irq = $irqmap->{$name}; + + syslog(LOG_INFO, "%s: queue %d assign %s to cpu %d", + $ifname, $q, $name, $cpu ); + + # Assign CPU affinity for both IRQs + set_affinity( $ifname, $irq, 1 << $cpu ); + + # TODO use RPS to steer data if cores > queues? + } + $cpu = next_cpu($cpu); + } +} + +# Affinity assignment function for single-queue NICs. The strategy +# here is to just spread the interrupts of different NICs evenly +# across all CPUs. That is the best we can do without monitoring the +# load and traffic patterns. So we just directly map the NIC unit +# number into a CPU number. +sub assign_single { + my ( $ifname, $irq ) = @_; + my $cpu = first_cpu($ifname, 1); + + syslog( LOG_INFO, "%s: assign irq %d to cpu %d", $ifname, $irq, $cpu ); + + set_affinity( $ifname, $irq, 1 << $cpu ); + + my $threads = $cpus / $cores; + if ($threads > 1) { + # Use both threads on this cpu if hyperthreading + my $mask = ((1 << $threads) - 1) << $cpu; + set_rps($ifname, 0, $mask); + } + # MAYBE - Use all cpu's if no HT +} + +# find irq number used for given interface using sysfs +sub get_irq { + my $ifname = shift; + + open( my $irqf, '<', "/sys/class/net/$ifname/device/irq" ) + or warn "$ifname: can't find irq : $!\n"; + my $irq = <$irqf>; + chomp $irq; + close $irqf; + + return $irq; +} + +# Mask must contain at least one CPU and +# no bits outside of range of available CPU's +sub check_mask { + my ($ifname, $name, $mask) = @_; + my $m = hex($mask); + + die "$ifname: $name mask $mask has no bits set\n" + if ($m == 0); + + die "$ifname: $name mask $mask to large for number of CPU's: $cpus\n" + if ($m >= 1 << $cpus); +} + +# Set affinity (and RPS) based on mask +sub affinity_mask { + my ($ifname, $mask) = @_; + + # match on <hex> or <hex>,<hex> + unless ($mask =~ /^([0-9a-f]+)(|,([0-9a-f]+))$/) { + die "$ifname: irq mask $mask is not a valid affinity mask\n" + } + + my $irq = $1; + my $rps = $3; + + check_mask($ifname, "irq", $irq); + check_mask($ifname, "rps", $rps) if $rps; + + set_affinity($ifname, get_irq($ifname), hex($irq)); + set_rps($ifname, 0, hex($rps)) if $rps; +} + +# The auto strategy involves trying to achieve the following goals: +# +# - Spread the receive load among as many CPUs as possible. +# +# - For all multi-queue NICs in the system that provide both tx and +# rx queues, keep all of the queues that share the same queue +# number on same CPUs. I.e. tx and rx queue 0 of all such NICs +# should interrupt one CPU; tx and rx queue 1 should interrupt a +# different CPU, etc. +# +# - If hyperthreading is supported and enabled, avoid assigning +# queues to both CPUs of a hyperthreaded pair if there are enough +# CPUs available to do that. +sub affinity_auto { + my $ifname = shift; + my $irqmap = irqinfo(); + my @irqnames = keys %{$irqmap}; + + # Figure out what style of irq naming is being used + my $numirq = grep { /$ifname/ } @irqnames; + if ( $numirq > 1 ) { + my $nq = grep { /$ifname-rx-/ } @irqnames; + + if ( $nq > 0 ) { + my $ntx = grep { /$ifname-tx-/ } @irqnames; + die "$ifname: rx queues $nq != tx queues $ntx" + unless ( $nq == $ntx ); + + return assign_multiqueue( $ifname, $nq, $irqmap, + [ '%s-rx-%d', '%s-tx-%d' ] ); + } + + $nq = grep { /$ifname-TxRx-/ } @irqnames; + if ( $nq > 0 ) { + return assign_multiqueue( $ifname, $nq, $irqmap, ['%s-TxRx-%d'] ); + } + + $nq = grep { /$ifname-\d$/ } @irqnames; + if ( $nq > 0 ) { + return assign_multiqueue( $ifname, $nq, $irqmap, ['%s-%d'] ); + } + + die "Unknown multiqueue device naming for $ifname\n"; + } + + assign_single( $ifname, get_irq($ifname) ); +} diff --git a/scripts/system/vyatta-auto-irqaffin.pl b/scripts/system/vyatta-auto-irqaffin.pl deleted file mode 100755 index 54b75638..00000000 --- a/scripts/system/vyatta-auto-irqaffin.pl +++ /dev/null @@ -1,467 +0,0 @@ -#!/usr/bin/perl -# -# Module: vyatta-auto-irqaffin.pl -# -# **** License **** -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# This code was originally developed by Vyatta, Inc. -# Portions created by Vyatta are Copyright (C) 2009,2010 Vyatta, Inc. -# All Rights Reserved. -# -# Author: Bob Gilligan (gilligan@vyatta.com) -# Date: October 2009 -# Description: Script to configure optimal IRQ affinity for NICs. -# -# **** End License **** -# - -# This script attempts to set up a static CPU affinity for the IRQs -# used by network interfaces. It is primarily targeted at supporting -# multi-queue NICs, but does include code to handle single-queue NICs. -# Since different NICs may have different queue organizations, and -# because there is no standard API for learning the mapping between -# queues and IRQ numbers, different code is required for each of the -# queue naming conventions. -# -# The general strategy involves trying to achieve the following goals: -# -# - Spread the receive load among as many CPUs as possible. -# -# - For all multi-queue NICs in the system that provide both tx and -# rx queues, keep all of the queues that share the same queue -# number on same CPUs. I.e. tx and rx queue 0 of all such NICs -# should interrupt one CPU; tx and rx queue 1 should interrupt a -# different CPU, etc. -# -# - If hyperthreading is supported and enabled, avoid assigning -# queues to both CPUs of a hyperthreaded pair if there are enough -# CPUs available to do that. -# -# This strategy yields the greatest MP scaling possible for -# multi-queue NICs. It also ensures that an individual skb is -# processed on the same CPU for the entirity of its lifecycle, -# including transmit time, which optimally utilizes the cache and -# keeps performance high. -# - - -use lib "/opt/vyatta/share/perl5"; -use Getopt::Long; - -use warnings; -use strict; - -# Send output of shell commands to syslog for debugging and so that -# the user is not confused by it. Log at debug level, which is supressed -# by default, so that we don't unnecessarily fill up the syslog file. -my $logger = 'logger -t firewall-cfg -p local0.debug --'; - -# Enable printing debug output to stdout. -my $debug_flag = 0; -my $syslog_flag = 0; - -my $setup_ifname; - -GetOptions("setup=s" => \$setup_ifname, - "debug" => \$debug_flag - ); - -sub log_msg { - my $message = shift; - - print "DEBUG: $message" if $debug_flag; - system("$logger DEBUG: \"$message\"") if $syslog_flag; -} - - -# Affinity assignment function for the Intel igb, ixgb and ixgbe -# drivers, and any other NICs that follow their queue naming -# convention. These NICs have an equal number of rx and tx queues. -# The first part of the strategy for optimal performance is to select -# the CPU to assign the IRQs to by mapping from the queue number. -# This ensures that all queues with the same queue number are assigned -# to the same CPU. The second part is to avoid assigning any queues -# to the second CPU in a hyper-threaded pair, if posible. I.e., if -# CPU 0 and 1 are hyper-threaded pairs, then assign a queue to CPU 0, -# but try to avoid assigning one to to CPU 1. But if we have more -# queues than CPUs, then it is OK to assign some to the second CPU in -# a hyperthreaded pair. -# -sub intel_func{ - my ($ifname, $numcpus, $numcores) = @_; - my $rx_queues; # number of rx queues - my $tx_queues; # number of tx queues - my $ht_factor; # 2 if HT enabled, 1 if not - my $start_cpu; # CPU number to start assignment at - - log_msg("intel_func was called.\n"); - - if ($numcpus > $numcores) { - $ht_factor = 2; - } else { - $ht_factor = 1; - } - - log_msg("ht_factor is $ht_factor.\n"); - - # Figure out how many queues we have - - $rx_queues=`grep "$ifname-rx-" /proc/interrupts | wc -l`; - $rx_queues =~ s/\n//; - - $tx_queues=`grep "$ifname-tx-" /proc/interrupts | wc -l`; - $tx_queues =~ s/\n//; - - log_msg("rx_queues is $rx_queues. tx_queues is $tx_queues\n"); - - if ($rx_queues != $tx_queues) { - printf("Error: rx and tx queues don't match for igb driver.\n"); - exit 1; - } - - - # Special case of a single-queue masquarading as a multi-queue NIC - if ($rx_queues == 1) { - $ifname =~ m/^eth(.*)$/; - - my $ifunit = $1; - log_msg ("ifunit = $ifunit\n"); - - if ($numcpus > $numcores) { - # Hyperthreaded - $start_cpu = (2 * $ifunit) % $numcpus; - - # every other time it wraps, add one to use the hyper-thread pair - # of the CPU selected. - my $use_ht = ((2 * $ifunit) / $numcpus) % 2; - $start_cpu += $use_ht; - } else { - # Not hyperthreaded. Map it to unit number MOD number of linux CPUs. - $start_cpu = $ifunit % $numcpus; - } - } else { - $start_cpu = 0; - } - - # For i = 0 to number of queues: - # Affinity of rx and tx queue $i gets CPU ($i * (2 if HT, 1 if no HT)) - # % number_of_cpus - for (my $queue = 0, my $cpu = $start_cpu; ($queue < $rx_queues) ; - $queue++) { - # Generate the hex string for the bitmask representing this CPU - my $cpu_bit = 1 << $cpu; - my $cpu_hex = sprintf("%x", $cpu_bit); - log_msg ("queue=$queue cpu=$cpu cpu_bit=$cpu_bit cpu_hex=$cpu_hex\n"); - - # Get the IRQ number for RX queue - my $rx_irq=`grep "$ifname-rx-$queue\$" /proc/interrupts | awk -F: '{print \$1}'`; - $rx_irq =~ s/\n//; - $rx_irq =~ s/ //g; - - # Get the IRQ number for TX queue - my $tx_irq=`grep "$ifname-tx-$queue\$" /proc/interrupts | awk -F: '{print \$1}'`; - $tx_irq =~ s/\n//; - $tx_irq =~ s/ //g; - - log_msg("rx_irq = $rx_irq. tx_irq = $tx_irq\n"); - - # Assign CPU affinity for both IRQs - system "echo $cpu_hex > /proc/irq/$rx_irq/smp_affinity"; - system "echo $cpu_hex > /proc/irq/$tx_irq/smp_affinity"; - - $cpu += $ht_factor; - - if ($cpu >= $numcpus) { - # Must "wrap" - $cpu %= $numcpus; - - if ($ht_factor > 1) { - # Next time through, select the other CPU in a hyperthreaded - # pair. - if ($cpu == 0) { - $cpu++; - } else { - $cpu--; - } - } - } - } -}; - - -# Affinity setting function for NICs using new intel queue scheme -# that provides one IRQ for each pair of TX and RX queues -sub intel_new_func{ - my ($ifname, $numcpus, $numcores) = @_; - my $txrx_queues; # number of rx/rx queue pairs - my $ht_factor; # 2 if HT enabled, 1 if not - - log_msg("intel_new_func was called.\n"); - - if ($numcpus > $numcores) { - $ht_factor = 2; - } else { - $ht_factor = 1; - } - - log_msg("ht_factor is $ht_factor.\n"); - - # Figure out how many queues we have - - $txrx_queues=`grep "$ifname-TxRx-" /proc/interrupts | wc -l`; - $txrx_queues =~ s/\n//; - - log_msg("txrx_queues is $txrx_queues.\n"); - - if ($txrx_queues <= 0) { - printf("Error: No TxRx queues found for new intel driver.\n"); - exit 1; - } - - # For i = 0 to number of queues: - # Affinity of TX/RX queue $i gets CPU ($i * (2 if HT, 1 if no HT)) - # % number_of_cpus - for (my $queue = 0, my $cpu = 0; ($queue < $txrx_queues) ; $queue++) { - # Generate the hex string for the bitmask representing this CPU - my $cpu_bit = 1 << $cpu; - my $cpu_hex = sprintf("%x", $cpu_bit); - log_msg ("queue=$queue cpu=$cpu cpu_bit=$cpu_bit cpu_hex=$cpu_hex\n"); - - # Get the IRQ number for RX queue - my $txrx_irq=`grep "$ifname-TxRx-$queue\$" /proc/interrupts | awk -F: '{print \$1}'`; - $txrx_irq =~ s/\n//; - $txrx_irq =~ s/ //g; - - log_msg("txrx_irq = $txrx_irq.\n"); - - # Assign CPU affinity for this IRQs - system "echo $cpu_hex > /proc/irq/$txrx_irq/smp_affinity"; - - $cpu += $ht_factor; - - if ($cpu >= $numcpus) { - # Must "wrap" - $cpu %= $numcpus; - - if ($ht_factor > 1) { - # Next time through, select the other CPU in a hyperthreaded - # pair. - if ($cpu == 0) { - $cpu++; - } else { - $cpu--; - } - } - } - } -}; - - -# Affinity assignment function for Broadcom NICs using the bnx2 driver -# or other multi-queue NICs that follow their queue naming convention. -# This strategy is similar to that for Intel drivers. But since -# Broadcom NICs do not have separate receive and transmit queues we -# perform one affinity assignment per queue. -# -sub broadcom_func{ - my ($ifname, $numcpus, $numcores) = @_; - my $num_queues; # number of queues - my $ht_factor; # 2 if HT enabled, 1 if not - - log_msg("broadcom_func was called.\n"); - - # Figure out how many queues we have - $num_queues=`egrep "$ifname\[-.\]\{1\}" /proc/interrupts | wc -l`; - $num_queues =~ s/\n//; - - log_msg("num_queues=$num_queues\n"); - - if ($num_queues <=0) { - printf("ERROR: No queues found for $ifname\n"); - exit 1; - } - - if ($numcpus > $numcores) { - $ht_factor = 2; - } else { - $ht_factor = 1; - } - - log_msg("ht_factor is $ht_factor.\n"); - - for (my $queue = 0, my $cpu = 0; ($queue < $num_queues) ; $queue++) { - # Generate the hex string for the bitmask representing this CPU - my $cpu_bit = 1 << $cpu; - my $cpu_hex = sprintf("%x", $cpu_bit); - log_msg ("queue=$queue cpu=$cpu cpu_bit=$cpu_bit cpu_hex=$cpu_hex\n"); - - # Get the IRQ number for the queue - my $irq=`egrep "$ifname\[-.fp\]*$queue\$" /proc/interrupts | awk -F: '{print \$1}'`; - $irq =~ s/\n//; - $irq =~ s/ //g; - - log_msg("irq = $irq.\n"); - - # Assign CPU affinity for this IRQs - system "echo $cpu_hex > /proc/irq/$irq/smp_affinity"; - - $cpu += $ht_factor; - if ($cpu >= $numcpus) { - # Must "wrap" - $cpu %= $numcpus; - - if ($ht_factor > 1) { - # Next time through, select the other CPU in a hyperthreaded - # pair. - if ($cpu == 0) { - $cpu++; - } else { - $cpu--; - } - } - } - } -} - - -# Affinity assignment function for single-quque NICs. The strategy -# here is to just spread the interrupts of different NICs evenly -# across all CPUs. That is the best we can do without monitoring the -# load and traffic patterns. So we just directly map the NIC unit -# number into a CPU number. -# -sub single_func { - my ($ifname, $numcpus, $numcores) = @_; - my $cpu; - use integer; - - log_msg("single_func was calledn.\n"); - - $ifname =~ m/^eth(.*)$/; - - my $ifunit = $1; - log_msg ("ifunit = $ifunit\n"); - - # Get the IRQ number for the queue - my $irq=`grep "$ifname" /proc/interrupts | awk -F: '{print \$1}'`; - $irq =~ s/\n//; - $irq =~ s/ //g; - - log_msg("irq = $irq.\n"); - - # Figure out what CPU to assign it to - if ($numcpus > $numcores) { - # Hyperthreaded - $cpu = (2 * $ifunit) % $numcpus; - - # every other time it wraps, add one to use the hyper-thread pair - # of the CPU selected. - my $use_ht = ((2 * $ifunit) / $numcpus) % 2; - $cpu += $use_ht; - } else { - # Not hyperthreaded. Map it to unit number MOD number of linux CPUs. - $cpu = $ifunit % $numcpus; - } - - # Generate the hex string for the bitmask representing this CPU - my $cpu_bit = 1 << $cpu; - my $cpu_hex = sprintf("%x", $cpu_bit); - log_msg ("cpu=$cpu cpu_bit=$cpu_bit cpu_hex=$cpu_hex\n"); - - # Assign CPU affinity for this IRQs - system "echo $cpu_hex > /proc/irq/$irq/smp_affinity"; -} - -# Mapping from driver type to function that handles it. -my %driver_hash = ( 'intel' => \&intel_func, - 'intel_new' => \&intel_new_func, - 'broadcom' => \&broadcom_func, - 'single' => \&single_func); - -if (defined $setup_ifname) { - # Set up automatic IRQ affinity for the named interface - - log_msg("setup $setup_ifname\n"); - - my $ifname = $setup_ifname; # shorter variable name - my $drivername; # Name of the NIC driver, e.g. "igb". - my $numcpus; # Number of Linux "cpus" - my $numcores; # Number of unique CPU cores - my $driver_func; # Pointer to fuction specific to a driver - my $driver_style; # Style of the driver. Whether it is multi-queue - # or not, and if it is, how it names its queues. - - # Determine how many CPUs the machine has. - $numcpus=`grep "^processor" /proc/cpuinfo | wc -l`; - $numcpus =~ s/\n//; - - log_msg("numcpus is $numcpus\n"); - - if ($numcpus == 1) { - # Nothing to do if we only have one CPU, so just exit quietly. - exit 0; - } - - # Determine how many cores the machine has. Could be less than - # the number of CPUs if processor supports hyperthreading. - $numcores=`grep "^core id" /proc/cpuinfo | uniq | wc -l`; - $numcores =~ s/\n//; - - log_msg("numcores is $numcores.\n"); - - # Verify that interface exists - if (! (-e "/proc/sys/net/ipv4/conf/$ifname")) { - printf("Error: Interface $ifname does not exist\n"); - exit 1; - } - - # Figure out what style of driver this NIC is using. - my $numints=`grep $ifname /proc/interrupts | wc -l`; - $numints =~ s/\n//; - if ($numints > 1) { - # It is a multiqueue NIC. Now figure out which one. - my $rx_queues=`grep "$ifname-rx-" /proc/interrupts | wc -l`; - $rx_queues =~ s/\n//; - if ($rx_queues > 0) { - # Driver is following the original Intel queue naming style - $driver_style="intel"; - } else { - my $rx_queues=`grep "$ifname-TxRx-" /proc/interrupts | wc -l`; - if ($rx_queues > 0) { - # Driver is following the new Intel queue naming - # style where on IRQ is used for each pair of - # TX and RX queues - $driver_style="intel_new"; - } else { - # The only other queue naming style that we have seen is the - # one used by Broadcom NICs. - $driver_style="broadcom"; - } - } - } elsif ($numints == 1) { - # It is a single queue NIC. - $driver_style="single"; - } else { - # $numints must be 0 - printf("Unable to determine IRQs for interface $ifname.\n"); - exit 0; - } - $driver_func = $driver_hash{$driver_style}; - - &$driver_func($ifname, $numcpus, $numcores); - - exit 0; -} - -printf("Must specify options.\n"); -exit(1); - - diff --git a/scripts/system/vyatta-irqaffin b/scripts/system/vyatta-irqaffin deleted file mode 100755 index 092da818..00000000 --- a/scripts/system/vyatta-irqaffin +++ /dev/null @@ -1,194 +0,0 @@ -#!/bin/bash - -# Author: Robert E. Gilligan <gilligan@vyatta.com> -# Date: 2008 -# Description: CLI back-end script to manipulate NIC interrupt CPU affinity. - -# **** License **** -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# This code was originally developed by Vyatta, Inc. -# Portions created by Vyatta are Copyright (C) 2006, 2007, 2008 Vyatta, Inc. -# All Rights Reserved. -# **** End License **** - -# Provides sub-commands to: -# - Check the validity of an affinity mask value -# - Set the affinity mask to the IRQs being used by an interface -# - Reset the affinity mask of the IRQs being used by an interface to the -# system default value of all-ones. -# - Print the affinity mask of the IRQs being used by an interface -# -# If the NIC in question supports multiple IRQs, the "set" sub-command -# sets all IRQs to the same mask. The "print" sub-command displays -# the mask of each IRQ individually. -# - -# Max number of hex characters in an IRQ affinity mask. Support up to 64 CPUs. -MAX_MASK=16 - -# Set up some global values... -numcpus=`grep -c -e "^processor" /proc/cpuinfo` -declare -i maxmask=(2**numcpus) -let maxmask=maxmask-1 -maxmaskhex=`printf "%x" ${maxmask}` - -print_usage() -{ - echo "Usage:" - echo -e "\t$0 check <ifname> <mask>" - echo -e "\t$0 set <ifname> <mask>" - echo -e "\t$0 reset <ifname>" - echo -e "\t$0 print <ifname>" -} - -get_irqnums() -{ - irqnums=`grep $1 /proc/interrupts | awk -F ': ' '{ print $1 }'` - if [ -z "$irqnums" ]; then - echo "Unable to determine IRQs for interface $1" - return 1 - fi - return 0 -} - - -get_mask() -{ - mask=$1 - - # mask must be a short hex value - if [ ${#mask} -gt $MAX_MASK ]; then - echo "mask too long: ${#2} characters." - return 1 - fi - - # strip out all the hex digits - exmask=`echo $mask | sed -e s/[0-9a-fA-F]//g` - - # if anything is left, its not hex - if [ ! -z "$exmask" ]; then - echo "Invalid characters in hex mask: $exmask" - return 1 - fi - - declare -i intmask=0x${mask} - - # Make sure that mask holds at least one bit, and holds no more bits - # than we have CPUs. - - if [ ${intmask} -eq 0 ]; then - echo "Mask can not be 0." - return 1 - fi - - if [ $intmask -gt $maxmask ]; then - echo "Mask is too large. Maximum hexidecimal bitmask is: ${maxmaskhex}" - return 1 - fi - - return 0 -} - - -case "$1" in - check) - # Note: We don't validate the interface name even though - # it is available as a command argument. That is because - # the interface may not exist or may not be configured at - # the time the check is performed. - # - if [ $# -ne 3 ]; then - print_usage - exit 1 - fi - - if ! get_mask $3 ; then - exit 1 - fi - exit 0 - ;; - - set) - if [ $# -ne 3 ]; then - print_usage - exit 1 - fi - - if ! get_irqnums $2 ; then - exit 1 - fi - - if ! get_mask $3 ; then - exit 1 - fi - - for irqnum in $irqnums ; do - echo $mask > /proc/irq/$irqnum/smp_affinity - done - - if [ $? -ne 0 ]; then - echo "Couldn't assign smp_affinity. Exit status: $?" - exit 1 - fi - ;; - - reset) - if [ $# -ne 2 ]; then - print_usage - exit 1 - fi - if ! get_irqnums $2 ; then - exit 1 - fi - - if [ -e /proc/irq/default_smp_affinity ]; then - defmask=`cat /proc/irq/default_smp_affinity` - else - defmask=$maxmaskhex - fi - - for irqnum in $irqnums ; do - echo $defmask > /proc/irq/$irqnum/smp_affinity - if [ $? -ne 0 ]; then - echo "Couldn't assign smp_affinity for IRQ $irqnum. Exit status: $?" - exit 1 - fi - done - ;; - - - print) - if [ $# -ne 2 ]; then - print_usage - exit 1 - fi - if ! get_irqnums $2 ; then - exit 1 - fi - - for irqnum in $irqnums ; do - mask=`cat /proc/irq/$irqnum/smp_affinity` - - if [ -z $mask ]; then - echo "Couldn't get smp_affinity for interface $2, irq $irqnum" - exit 1 - fi - - echo "Interface: $2 IRQ: $irqnum Mask: $mask" - done - ;; - - *) - print_usage - exit 1 - ;; - -esac diff --git a/templates/interfaces/ethernet/node.tag/smp_affinity/node.def b/templates/interfaces/ethernet/node.tag/smp_affinity/node.def index 671a28f0..1f8a2610 100644 --- a/templates/interfaces/ethernet/node.tag/smp_affinity/node.def +++ b/templates/interfaces/ethernet/node.tag/smp_affinity/node.def @@ -15,24 +15,14 @@ type: txt help: CPU interrupt affinity mask -val_help: <hex>; Bitmask representing CPUs that this NIC will interrupt val_help: auto; Automatic CPU affinity (default) +val_help: <hex>; Bitmask representing CPUs that this NIC will interrupt +val_help: <hex>,<hex>; Bitmasks representing CPUs for interrupt and receive processing default: "auto" -syntax:expression: $VAR(@) == "auto" || pattern $VAR(@) "^[0-9a-f]+$" ; +syntax:expression: $VAR(@) == "auto" || pattern $VAR(@) "^[0-9a-f]+(|,[0-9a-f]+)$" ; "IRQ affinity mask must be hex value or auto" -commit:expression: exec "\ - [ \"$VAR(@)\" == \"auto\" ] || \ - /opt/vyatta/sbin/vyatta-irqaffin check $VAR(../@) $VAR(@)" - -update: if [ "$VAR(@)" = "auto" ]; then - sudo /opt/vyatta/sbin/vyatta-auto-irqaffin.pl --setup $VAR(../@) - else - sudo /opt/vyatta/sbin/vyatta-irqaffin set $VAR(../@) $VAR(@) - fi - -delete: [ -d /sys/class/net/$VAR(../@) ] || exit 0 - sudo /opt/vyatta/sbin/vyatta-irqaffin reset $VAR(../@) +update: sudo /opt/vyatta/sbin/irq-affinity.pl $VAR(../@) $VAR(@) |