WL#5960: Start memcached from mysql-test-run.pl

Status: Complete   —   Priority: Medium

The mysql-test-run.pl script will be modified so that it is able to start and 
restart a memcached server.  This will be used for testing of NDB+Memcached.


This describes the design as implemented in the MySQL Cluster 7.2 release. 

(1) It is standard to build MySQL Cluster using an in-tree Memcached, but also
possible to build against an existing memcached outside the MySQL source tree. 
In either case CMake is used at configure time to generate a perl file which
will point mtr to the correct runnable memcached binary.  CMake uses the
template file storage/ndb/memcache/memcached_path.pl.in to generate the Perl
file memcached_path.pl.

(2) After starting MySQL Cluster, but before starting memcached, the SQL script
ndb_memcache_metadata.sql must be run.  A new perl function
memcached_load_metadata() in mtr locates the runs this script.

(3) A new memcached_start() function in mtr reads the memcached_path.pl file to
find memcached, also finds the ndb_engine.so file specific to NDB+Memcached, and
starts memcached.  Memcached is always started on port "$baseport+8" -- 8 ports
above mtr's base port. 

(4) Finally, a new memcache client library for Perl is added at mysql-
test/lib/My/Memcache.pm.  This library is for use by mtr test cases.  It supports 
all basic memcache operations using both the text and binary protocols over TCP, 
and it could also be applicable to Memcached with InnoDB. 
Here is the patch as currently implemented in MySQL Cluster 7.2.

247a248
> my $memcached_base_port;
1805a1807
>   $memcached_base_port = $baseport + 8;
2991a2994,3121
> sub memcached_start {
>   my ($cluster, $memcached, $port_number) = @_;
>   
>   mtr_verbose("memcached_start"); 
>   
>   my $found_perl_source = my_find_file($basedir, 
>      ["storage/ndb/memcache",        # source
>       "mysql-test/lib",              # install
>       "share/mysql-test/lib"],       # install 
>       "memcached_path.pl", NOT_REQUIRED);
>   
>   my $found_so = my_find_file($bindir,
>     ["storage/ndb/memcache/",       # source or build
>      "lib", "lib64"],               # install
>     "ndb_engine.so", NOT_REQUIRED); 
> 
>   mtr_verbose("Found memcache script: $found_perl_source");
>   mtr_verbose("Found memcache plugin: $found_so");
>      
>   my $mgm_host;
>   my $mgm_port;
>   foreach my $mgmd ( in_cluster($cluster, ndb_mgmds()) ) {
>     $mgm_host = $mgmd->value('HostName');
>     $mgm_port = $mgmd->value('PortNumber');
>     last;
>   }
>   my $ndb_opt_string = "connectstring=$mgm_host:$mgm_port";
>   my $options = $memcached->value("options");
>   if($options) {  
>     $ndb_opt_string = $ndb_opt_string . ";" . $options;
>   }
> 
> 
>   $found_perl_source ne "" or return;
>   $found_so ne "" or mtr_error("Failed to find ndb_engine.so");  
>   require "$found_perl_source";
>   if(! memcached_is_available()) 
>   {
>     mtr_error("Memcached not available.");
>   }
>   my $exe = "";
>   if(memcached_is_bundled())
>   {
>     $exe = my_find_bin($bindir,
>     ["libexec", "sbin", "bin", "storage/ndb/memcache/extra/memcached"],
>     "memcached", NOT_REQUIRED);
>   }
>   else 
>   {
>     $exe = get_memcached_exe_path();
>   }
>   $exe ne "" or mtr_error("Failed to find memcached.");
> 
>   my $args;
>   mtr_init_args(\$args);
>   mtr_add_arg($args, "-p");
>   mtr_add_arg($args, $port_number);
>   mtr_add_arg($args, "-c");
>   mtr_add_arg($args,"100");   # max 100 connections
>   mtr_add_arg($args, "-E");
>   mtr_add_arg($args, $found_so);  # /path/ndb_engine.so    
>   mtr_add_arg($args, "-e");  
>   mtr_add_arg($args, "$ndb_opt_string");
> 
>   if($opt_gdb)
>   {
>     gdb_arguments(\$args, \$exe, "memcached");
>   }
>   
>   my $proc = My::SafeProcess->new 
>   ( name     =>  "memcached",
>     path     =>  $exe,
>     args     => \$args,
>     output   =>  "$opt_vardir/log/memcached.out",
>     error    =>  "$opt_vardir/log/memcached.out",
>     append   =>  1,
>   );
>   mtr_verbose("Started $proc");
>   
>   $memcached->{proc} = $proc;
>   
>   return;  
> }
> 
> 
> sub memcached_load_metadata($) {
>   my $cluster= shift;
>     
>   my $sql_script= my_find_file($bindir,
>                               ["share/mysql/memcache-api", # RPM install
>                                "share/memcache-api",       # Other installs
>                                "scripts"                   # Build tree
>                               ],
>                               "ndb_memcache_metadata.sql", NOT_REQUIRED);
> 
>   foreach my $mysqld (mysqlds()) {
>     if(-d $mysqld->value('datadir') . "/" . "ndbmemcache") {
>       mtr_verbose("skipping memcache metadata (already stored)");
>       return;
>     }
>   }
>   
>   mtr_verbose("memcached_load_metadata: $sql_script");
>   
>   if (-f $sql_script )
>   {
>     my $args;
>     mtr_init_args(\$args);
>     mtr_add_arg($args, "--defaults-file=%s", $path_config_file);
>     mtr_add_arg($args, "--defaults-group-suffix=%s", $cluster->suffix());
>     mtr_add_arg($args, "--connect-timeout=20");
>     
>     mtr_verbose("Script: $sql_script");
>     if ( My::SafeProcess->run(
>            name   => "ndbmemcache config loader",
>            path   => $exe_mysql,
>            args   => \$args,
>            input  => $sql_script,
>            output => "$opt_vardir/log/memcache_config.log",
>            error  => "$opt_vardir/log/memcache_config.log"
>        ) != 0)
>     {
>       mtr_error("Could not load ndb_memcache_metadata.sql file");      
>     }
>   }
> }
> 
> 
5142c5272,5273
< sub all_servers { return ( mysqlds(), ndb_mgmds(), ndbds() ); }
---
> sub memcacheds { return _like('memcached.'); }
> sub all_servers { return ( mysqlds(), ndb_mgmds(), ndbds(), memcacheds() ); }
5215c5346
< 			       started(ndbds(), ndb_mgmds()) );
---
> 			       started(ndbds(), ndb_mgmds(), memcacheds()) );
5396a5528,5556
>   
>   # Start memcached if needed
>   foreach my $cluster (clusters()) 
>   { 
>     if(memcacheds()) 
>     {
>       # In theory maybe you could run the memcached tests with the embedded
>       # server, but for now we skip it.
>       if($opt_embedded_server) 
>       { 
>         mtr_error("Cannot run memcached tests with the embedded server.");
>       }
>       
>       my $avail_port = $memcached_base_port;
>       my $memcached;
>       memcached_load_metadata($cluster);
> 
>       # Start them
>       foreach $memcached( in_cluster($cluster, memcacheds()) )
>       {
>         if(! started($memcached)) 
>         {
>           memcached_start($cluster, $memcached, $avail_port);
>         }
>         $avail_port += 1;
>       }
>     }
>   }
>